Skip to content

Instantly share code, notes, and snippets.

Show Gist options
  • Save dbeattie71/951946bbfaebba33ea902bb22f790243 to your computer and use it in GitHub Desktop.
Save dbeattie71/951946bbfaebba33ea902bb22f790243 to your computer and use it in GitHub Desktop.
A decompresing INntpConnection implementation (not very good)
using System;
using System.Collections.Generic;
using System.IO;
using System.Net.Security;
using System.Net.Sockets;
using System.Threading;
using System.Threading.Tasks;
using ICSharpCode.SharpZipLib.Zip.Compression;
using ICSharpCode.SharpZipLib.Zip.Compression.Streams;
using Microsoft.Extensions.Logging;
using Usenet.Exceptions;
using Usenet.Extensions;
using Usenet.Nntp.Parsers;
using Usenet.Nntp.Responses;
using Usenet.Util;
using Usenet.Yenc;
namespace Usenet.Nntp
{
public class NntpDecompressingConnection : INntpConnection
{
private static readonly ILogger log = Logger.Create<NntpDecompressingConnection>();
private const int bufferSize = 4096; // 8192
private const int yencLineBufferSize = 1024;
private const int readTimeout = 500; // msec
private const int lfByte = 10;
private const int crByte = 13;
private const int spaceByte = 32;
private const int dotByte = 46;
private const int zeroByte = 48;
private const int nineByte = 57;
private static readonly byte[] yBeginBytes = UsenetEncoding.Default.GetBytes(YencKeywords.Header);
private static readonly byte[] yEndBytes = UsenetEncoding.Default.GetBytes(YencKeywords.Footer);
private readonly byte[] buffer = new byte[bufferSize];
private readonly byte[] yencLineBuffer = new byte[yencLineBufferSize];
private readonly TcpClient client = new TcpClient();
private readonly MemoryStream readBufferStream = new MemoryStream();
private readonly MemoryStream copyStream = new MemoryStream();
private readonly MemoryStream yencStream = new MemoryStream();
private Stream nntpStream;
private StreamWriter writer;
public async Task<TResponse> ConnectAsync<TResponse>(string hostname, int port, bool useSsl, IResponseParser<TResponse> parser)
{
log.LogInformation("Connecting: {hostname} {port} (Use SSl = {useSsl})", hostname, port, useSsl);
await client.ConnectAsync(hostname, port);
nntpStream = await GetStreamAsync(hostname, useSsl);
writer = new StreamWriter(nntpStream, UsenetEncoding.Default) { AutoFlush = true };
return GetResponse(parser);
}
public TResponse Command<TResponse>(string command, IResponseParser<TResponse> parser)
{
ThrowIfNotConnected();
log.LogInformation("Sending command: {Command}",
command.StartsWith("AUTHINFO PASS", StringComparison.Ordinal) ? "AUTHINFO PASS [omitted]" : command);
// send command and get response
writer.WriteLine(command);
return GetResponse(parser);
}
public TResponse MultiLineCommand<TResponse>(string command, IMultiLineResponseParser<TResponse> parser)
{
return MultiLineCommand(command, false, parser);
}
public TResponse DecompressingMultiLineCommand<TResponse>(string command, IMultiLineResponseParser<TResponse> parser)
{
return MultiLineCommand(command, true, parser);
}
private TResponse MultiLineCommand<TResponse>(string command, bool mustDecompress, IMultiLineResponseParser<TResponse> parser)
{
// send command
NntpResponse response = Command(command, new ResponseParser());
mustDecompress = mustDecompress || response.Message.Contains("COMPRESS", StringComparison.OrdinalIgnoreCase);
// receive multiline datablock (decompress stream if needed)
List<string> dataBlock = response.Success
? ReadMultiLineDataBlock(mustDecompress ? Decompress(nntpStream) : nntpStream)
: EmptyList<string>.Instance;
// create response object
return parser.Parse(response.Code, response.Message, dataBlock);
}
public TResponse GetResponse<TResponse>(IResponseParser<TResponse> parser)
{
string responseText = ReadLine(nntpStream, true);
log.LogInformation("Response received: {Response}", responseText);
if (responseText == null)
{
throw new NntpException("Received no response.");
}
if (responseText.Length < 3 || !int.TryParse(responseText.Substring(0, 3), out int code))
{
throw new NntpException("Received invalid response.");
}
return parser.Parse(code, responseText.Substring(4));
}
public void WriteLine(string line)
{
ThrowIfNotConnected();
writer.WriteLine(line);
}
private void ThrowIfNotConnected()
{
if (!client.Connected)
{
throw new NntpException("Client not connected.");
}
}
/// <summary>
/// Read line from binary stream.
/// Source: https://stackoverflow.com/questions/18358435/how-can-i-use-the-deflatestream-class-on-one-line-in-a-file
/// </summary>
/// <param name="source"></param>
/// <param name="mustSkipGarbage"></param>
/// <returns></returns>
private string ReadLine(Stream source, bool mustSkipGarbage)
{
readBufferStream.SetLength(0);
int next;
var offset = 0;
var foundPrefix = false;
var wasCr = false;
var garbageCount = 0;
var digitCount = 0;
while ((next = source.ReadByte()) >= 0)
{
if (next == lfByte && wasCr)
{
// CRLF found
// end of line (minus the CR)
return UsenetEncoding.Default.GetString(readBufferStream.ToArray(), offset, (int)readBufferStream.Length - offset -1);
}
if (mustSkipGarbage && !foundPrefix)
{
if (next == spaceByte && digitCount >= 3)
{
// found starting prefix: a 3 digit code + 1 space
offset = (int) readBufferStream.Length - 3;
foundPrefix = true;
if (garbageCount > 3)
{
log.LogWarning("Skipped garbage: {GarbageCount} bytes", garbageCount - 3);
}
}
else if (next < zeroByte || next > nineByte)
{
digitCount = 0;
}
else
{
digitCount++;
}
garbageCount++;
}
readBufferStream.WriteByte((byte)next);
wasCr = next == crByte;
}
// end of file
return readBufferStream.Length == 0
? null
: UsenetEncoding.Default.GetString(readBufferStream.ToArray(), offset, (int) readBufferStream.Length - offset);
}
private static int ReadLineBytes(Stream source, byte[] lineBuffer)
{
int next;
int lineBufferSize = lineBuffer.Length;
var lineIndex = 0;
var wasCr = false;
while ((next = source.ReadByte()) >= 0)
{
if (next == lfByte && wasCr)
{
// CRLF found
// end of line (minus the CR)
return lineIndex - 1;
}
if (lineIndex >= lineBufferSize)
{
// buffer full
return lineBufferSize;
}
lineBuffer[lineIndex++] = (byte)next;
wasCr = next == crByte;
}
// end of file
return lineIndex;
}
private async Task<Stream> GetStreamAsync(string hostname, bool useSsl)
{
NetworkStream stream = client.GetStream();
if (!useSsl)
{
return stream;
}
var sslStream = new SslStream(stream);
await sslStream.AuthenticateAsClientAsync(hostname);
return sslStream;
}
private List<string> ReadMultiLineDataBlock(Stream readStream)
{
var lines = new List<string>();
string line;
while ((line = ReadLine(readStream, false)) != null)
{
if (line == ".")
{
break;
}
lines.Add(line.StartsWith("..") ? line.Substring(1) : line);
}
return lines;
}
private Stream Decompress(Stream source)
{
// reset buffer
copyStream.SetLength(0);
// cannot stream but need to copy entire data block into memory because
// protocol does not work with a termintator
var byteCount = (int) CopyBinaryData(source, copyStream);
if (byteCount == 0)
{
return null;
}
// astraweb uses yenc compression
// source: http://helpdesk.astraweb.com/index.php?_m=news&_a=viewnews&newsid=9
// just deflate if no yenc encoding is used
copyStream.Position = 0;
copyStream.Read(buffer, 0, yBeginBytes.Length);
copyStream.Position = 0;
if (!buffer.StartsWith(yBeginBytes))
{
return new InflaterInputStream(copyStream, new Inflater(false), byteCount);
}
// skip yenc header line
ReadLineBytes(copyStream, yencLineBuffer);
// reset yenc buffer
yencStream.SetLength(0);
while ((byteCount = ReadLineBytes(copyStream, yencLineBuffer)) > 0)
{
//log.LogDebug(UsenetEncoding.Default.GetString(lineBuffer, 0, byteCount));
var offset = 0;
if (yencLineBuffer[0] == dotByte)
{
if (byteCount == 1)
{
break;
}
if (yencLineBuffer[1] == dotByte)
{
offset = 1;
}
}
else
{
if (yencLineBuffer.StartsWith(yEndBytes))
{
break;
}
}
int decodedCount = YencLineDecoder.Decode(yencLineBuffer, offset, byteCount - offset, buffer, 0);
yencStream.Write(buffer, 0, decodedCount);
}
// deflate
yencStream.Position = 0;
return new InflaterInputStream(yencStream, new Inflater(true), (int)yencStream.Length);
}
private long CopyBinaryData(Stream source, Stream target)
{
try
{
int byteCount;
source.ReadTimeout = readTimeout;
while ((byteCount = source.Read(buffer, 0, bufferSize)) > 0)
{
target.Write(buffer, 0, byteCount);
}
return target.Length;
}
catch (IOException)
{
// swallow
return target.Length;
}
finally
{
source.ReadTimeout = Timeout.Infinite;
}
}
public void Dispose()
{
client?.Dispose();
writer?.Dispose();
readBufferStream?.Dispose();
nntpStream?.Dispose();
}
}
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment