Quantcast
Channel: VBForums - Visual Basic .NET
Viewing all articles
Browse latest Browse all 27201

VS 2010 [RESOLVED] Uploading files in chunks with multiple threads (Producer/Consumer)

$
0
0
Hi, I'm creating a console application and I'm looking for the best way to upload files in chunks to a server with multiple connections (10 connections maximum). The files are uploaded as strings and not bytes. All files in a folder need to be read/encoded in a separate thread and added to a thread-safe queue. Each upload connection grabs a chunk from the queue and uploads it to the server, etc, etc, until all files have been uploaded ==> "Producer/Consumer". I don't want to read all data at once, because the files might be a few hundred megabytes.

Steps.

Code:

1) Create a separate thread to read chunks from all files in a folder.

2) Encode the chunks to string (custom encoder) and add them to a queue.

3) Temporarily halt this process when a certain number of items are in the queue.

4) Establish several multi-threaded connections to the server.

5) Each connection grabs an item from the queue and uploads it.

6) Continue reading/encoding chunks until a certain number of items are in the queue again.

7) When a chunk has been uploaded, then grab the next one and upload it.

8) Repeat steps 6 and 7 until all files have been uploaded.

Before spending more time on this I'd like to know what the best way is to do this, because I have a feeling that there's a better way.

Also, what if the uploading goes faster than the file reading/encoding? What is the proper way of letting the upload connections wait until there are items in the queue again?

vb.net Code:
  1. Imports System.Threading
  2. Imports System.Collections.Concurrent
  3. Imports System.IO
  4.  
  5. Module Program
  6.  
  7.     Public _queue As New ConcurrentQueue(Of String)
  8.     Public _waitHandle As New AutoResetEvent(False)
  9.  
  10.     Sub Main()
  11.  
  12.         ThreadPool.QueueUserWorkItem(New WaitCallback(AddressOf ReadEncodeFiles), "C:\Folder")
  13.  
  14.         For i As Integer = 1 To 5
  15.             Dim c As New Connections("server.com", 1500)
  16.         Next
  17.  
  18.         'Wait here.
  19.     End Sub
  20.  
  21.     Private Sub ReadEncodeFiles(ByVal state As Object)
  22.         Dim folderPath = DirectCast(state, String)
  23.  
  24.         Const BUFFER_LENGTH As Integer = 500 * 1024
  25.         Dim buffer(BUFFER_LENGTH - 1) As Byte
  26.  
  27.         For Each filePath In Directory.GetFiles(folderPath)
  28.             If File.Exists(filePath) Then
  29.                 Using file As New FileStream(filePath, FileMode.Open)
  30.                     Dim byteCount As Integer = file.Read(buffer, 0, BUFFER_LENGTH)
  31.  
  32.                     Do Until byteCount = 0
  33.  
  34.                         'Encode chunk to a string with custom encoder.
  35.                         Dim encodedData As String = EncodeToString(buffer.Take(byteCount).ToArray())
  36.  
  37.                         'Add encoded chunk to the queue.
  38.                         _queue.Enqueue(encodedData)
  39.  
  40.                         'Wait if there are 20 or more items in the queue.
  41.                         If (_queue.Count >= 20) Then
  42.                             _waitHandle.WaitOne()
  43.                         End If
  44.  
  45.                         byteCount = file.Read(buffer, 0, BUFFER_LENGTH)
  46.                     Loop
  47.                 End Using
  48.             End If
  49.         Next
  50.     End Sub
  51. End Module

vb.net Code:
  1. Imports System.Net.Sockets
  2. Imports System.IO
  3.  
  4. Public Class Connections
  5.  
  6.     Private _socket As TcpClient
  7.     Private _stream As Stream
  8.     Private _databuffer(1023) As Byte
  9.  
  10.     Public Sub New(ByVal server As String, _
  11.                    ByVal port As Integer)
  12.  
  13.         _socket = New TcpClient
  14.         _socket.SendTimeout = (30 * 1000)
  15.         _socket.ReceiveTimeout = (30 * 1000)
  16.         _socket.BeginConnect(server, port, AddressOf ServerConnect, Nothing)
  17.     End Sub
  18.  
  19.     Private Sub ServerConnect(ByVal ar As IAsyncResult)
  20.         Try
  21.             _socket.EndConnect(ar)
  22.             _stream = _socket.GetStream()
  23.             _stream.BeginRead(_databuffer, 0, 1024, AddressOf Read, _databuffer)
  24.         Catch ex As SocketException
  25.  
  26.         End Try
  27.     End Sub
  28.  
  29.     Private Sub Read(ByVal ar As IAsyncResult)
  30.         Try
  31.             If _stream IsNot Nothing Then
  32.  
  33.                 Dim byteCount = _stream.EndRead(ar)
  34.  
  35.                 If byteCount = 0 Then
  36.                     'Server closed the connection.
  37.                 Else
  38.  
  39.                     'Communication with server not added yet.
  40.  
  41.  
  42.  
  43.                     'Upload the next chunk of data from the queue.
  44.                     Dim Chunk As String = String.Empty
  45.                     If (_queue.TryDequeue(Chunk)) Then
  46.                         'Send next chunk of data.
  47.                         Send(Chunk)
  48.                         'Signal 'ReadEncodeFiles' to continue reading/encoding chunks of data.
  49.                         _waitHandle.Set()
  50.                     Else
  51.                         'Queue empty ??
  52.                     End If
  53.  
  54.  
  55.                     _stream.BeginRead(_databuffer, 0, 1024, AddressOf Read, _databuffer)
  56.                 End If
  57.             End If
  58.         Catch ex As Exception
  59.  
  60.         End Try
  61.     End Sub
  62.  
  63.     Private Sub Send(ByVal message As String)
  64.         Dim buffer As Byte() = System.Text.Encoding.GetEncoding("iso-8859-1").GetBytes(message) '1252
  65.  
  66.         Try
  67.             _stream.BeginWrite(buffer, 0, buffer.Length, AddressOf Write, Nothing)
  68.         Catch ioex As IOException
  69.  
  70.         End Try
  71.     End Sub
  72.  
  73.     Private Sub Write(ByVal ar As IAsyncResult)
  74.         _stream.EndWrite(ar)
  75.     End Sub
  76. End Class

Viewing all articles
Browse latest Browse all 27201

Trending Articles



<script src="https://jsc.adskeeper.com/r/s/rssing.com.1596347.js" async> </script>