Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[release/7.0] Fix HTTP/3 and HTTP/2 header decoder buffer allocation #47949

Merged
merged 4 commits into from
May 13, 2023
Merged

[release/7.0] Fix HTTP/3 and HTTP/2 header decoder buffer allocation #47949

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
e1f11dc
Fix HTTP/3 and HTTTP/2 header decoder buffer allocation
ManickaP Apr 28, 2023
011a87d
Fix warnings in H/QPack tests.
adityamandaleeka May 5, 2023
70e7028
Fix Asserts
ManickaP May 11, 2023
64df8d4
Fix test
ManickaP May 12, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 10 additions & 2 deletions .editorconfig
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -133,10 +133,12 @@ dotnet_diagnostic.CA1829.severity = warning
dotnet_diagnostic.CA1830.severity = warning

# CA1831: Use AsSpan or AsMemory instead of Range-based indexers when appropriate
# CA1832: Use AsSpan or AsMemory instead of Range-based indexers when appropriate
# CA1833: Use AsSpan or AsMemory instead of Range-based indexers when appropriate
dotnet_diagnostic.CA1831.severity = warning

# CA1832: Use AsSpan or AsMemory instead of Range-based indexers when appropriate
dotnet_diagnostic.CA1832.severity = warning

# CA1833: Use AsSpan or AsMemory instead of Range-based indexers when appropriate
dotnet_diagnostic.CA1833.severity = warning

# CA1834: Consider using 'StringBuilder.Append(char)' when applicable
Expand Down Expand Up @@ -290,6 +292,12 @@ dotnet_diagnostic.CA1826.severity = suggestion
dotnet_diagnostic.CA1827.severity = suggestion
# CA1829: Use Length/Count property instead of Count() when available
dotnet_diagnostic.CA1829.severity = suggestion
# CA1831: Use AsSpan or AsMemory instead of Range-based indexers when appropriate
dotnet_diagnostic.CA1831.severity = suggestion
# CA1832: Use AsSpan or AsMemory instead of Range-based indexers when appropriate
dotnet_diagnostic.CA1832.severity = suggestion
# CA1833: Use AsSpan or AsMemory instead of Range-based indexers when appropriate
dotnet_diagnostic.CA1833.severity = suggestion
# CA1834: Consider using 'StringBuilder.Append(char)' when applicable
dotnet_diagnostic.CA1834.severity = suggestion
# CA1835: Prefer the 'Memory'-based overloads for 'ReadAsync' and 'WriteAsync'
Expand Down
11 changes: 6 additions & 5 deletions src/Shared/runtime/Http2/Hpack/HPackDecoder.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -187,12 +187,11 @@ private void DecodeInternal(ReadOnlySpan<byte> data, IHttpStreamHeadersHandler h
// will no longer be valid.
if (_headerNameRange != null)
{
EnsureStringCapacity(ref _headerNameOctets);
EnsureStringCapacity(ref _headerNameOctets, _headerNameLength);
_headerName = _headerNameOctets;

ReadOnlySpan<byte> headerBytes = data.Slice(_headerNameRange.GetValueOrDefault().start, _headerNameRange.GetValueOrDefault().length);
headerBytes.CopyTo(_headerName);
_headerNameLength = headerBytes.Length;
_headerNameRange = null;
}
}
Expand Down Expand Up @@ -427,6 +426,7 @@ private void ParseHeaderName(ReadOnlySpan<byte> data, ref int currentIndex, IHtt
{
// Fast path. Store the range rather than copying.
_headerNameRange = (start: currentIndex, count);
_headerNameLength = _stringLength;
currentIndex += count;

_state = State.HeaderValueLength;
Expand Down Expand Up @@ -621,11 +621,12 @@ int Decode(ref byte[] dst)
_state = nextState;
}

private void EnsureStringCapacity(ref byte[] dst)
private void EnsureStringCapacity(ref byte[] dst, int stringLength = -1)
{
if (dst.Length < _stringLength)
stringLength = stringLength >= 0 ? stringLength : _stringLength;
if (dst.Length < stringLength)
{
dst = new byte[Math.Max(_stringLength, Math.Min(dst.Length * 2, _maxHeadersLength))];
dst = new byte[Math.Max(stringLength, Math.Min(dst.Length * 2, _maxHeadersLength))];
}
}

Expand Down
4 changes: 2 additions & 2 deletions src/Shared/runtime/Http3/QPack/QPackDecoder.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -243,12 +243,11 @@ private void DecodeInternal(ReadOnlySpan<byte> data, IHttpStreamHeadersHandler h
// will no longer be valid.
if (_headerNameRange != null)
{
EnsureStringCapacity(ref _headerNameOctets, _stringLength, existingLength: 0);
EnsureStringCapacity(ref _headerNameOctets, _headerNameLength, existingLength: 0);
_headerName = _headerNameOctets;

ReadOnlySpan<byte> headerBytes = data.Slice(_headerNameRange.GetValueOrDefault().start, _headerNameRange.GetValueOrDefault().length);
headerBytes.CopyTo(_headerName);
_headerNameLength = headerBytes.Length;
_headerNameRange = null;
}
}
Expand Down Expand Up @@ -294,6 +293,7 @@ private void ParseHeaderName(ReadOnlySpan<byte> data, ref int currentIndex, IHtt
{
// Fast path. Store the range rather than copying.
_headerNameRange = (start: currentIndex, count);
_headerNameLength = _stringLength;
currentIndex += count;

_state = State.HeaderValueLength;
Expand Down
110 changes: 106 additions & 4 deletions src/Shared/test/Shared.Tests/runtime/Http2/HPackDecoderTest.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,8 +46,13 @@ public class HPackDecoderTests

private const string _headerNameString = "new-header";

// On purpose longer than 4096 (DefaultStringOctetsSize from HPackDecoder) to trigger https://github.com/dotnet/runtime/issues/78516
private static readonly string _literalHeaderNameString = string.Concat(Enumerable.Range(0, 4100).Select(c => (char)('a' + (c % 26))));

private static readonly byte[] _headerNameBytes = Encoding.ASCII.GetBytes(_headerNameString);

private static readonly byte[] _literalHeaderNameBytes = Encoding.ASCII.GetBytes(_literalHeaderNameString);

// n e w - h e a d e r *
// 10101000 10111110 00010110 10011100 10100011 10010000 10110110 01111111
private static readonly byte[] _headerNameHuffmanBytes = new byte[] { 0xa8, 0xbe, 0x16, 0x9c, 0xa3, 0x90, 0xb6, 0x7f };
Expand All @@ -64,6 +69,12 @@ public class HPackDecoderTests
.Concat(_headerNameBytes)
.ToArray();

// size = 4096 ==> 0x7f, 0x81, 0x1f (7+) prefixed integer
// size = 4100 ==> 0x7f, 0x85, 0x1f (7+) prefixed integer
private static readonly byte[] _literalHeaderName = new byte[] { 0x7f, 0x85, 0x1f } // 4100
.Concat(_literalHeaderNameBytes)
.ToArray();

private static readonly byte[] _headerNameHuffman = new byte[] { (byte)(0x80 | _headerNameHuffmanBytes.Length) }
.Concat(_headerNameHuffmanBytes)
.ToArray();
Expand Down Expand Up @@ -392,6 +403,101 @@ public void DecodesLiteralHeaderFieldNeverIndexed_IndexedName_OutOfRange_Error()
Assert.Empty(_handler.DecodedHeaders);
}

[Fact]
public void DecodesLiteralHeaderFieldNeverIndexed_NewName_SingleBuffer()
{
byte[] encoded = _literalHeaderFieldWithoutIndexingNewName
.Concat(_literalHeaderName)
.Concat(_headerValue)
.ToArray();

_decoder.Decode(encoded, endHeaders: true, handler: _handler);

Assert.Single(_handler.DecodedHeaders);
Assert.True(_handler.DecodedHeaders.ContainsKey(_literalHeaderNameString));
Assert.Equal(_headerValueString, _handler.DecodedHeaders[_literalHeaderNameString]);
}

[Fact]
public void DecodesLiteralHeaderFieldNeverIndexed_NewName_NameLengthBrokenIntoSeparateBuffers()
{
byte[] encoded = _literalHeaderFieldWithoutIndexingNewName
.Concat(_literalHeaderName)
.Concat(_headerValue)
.ToArray();

_decoder.Decode(encoded[..1], endHeaders: false, handler: _handler);
_decoder.Decode(encoded[1..], endHeaders: true, handler: _handler);

Assert.Single(_handler.DecodedHeaders);
Assert.True(_handler.DecodedHeaders.ContainsKey(_literalHeaderNameString));
Assert.Equal(_headerValueString, _handler.DecodedHeaders[_literalHeaderNameString]);
}

[Fact]
public void DecodesLiteralHeaderFieldNeverIndexed_NewName_NameBrokenIntoSeparateBuffers()
{
byte[] encoded = _literalHeaderFieldWithoutIndexingNewName
.Concat(_literalHeaderName)
.Concat(_headerValue)
.ToArray();

_decoder.Decode(encoded[..(_literalHeaderNameString.Length / 2)], endHeaders: false, handler: _handler);
_decoder.Decode(encoded[(_literalHeaderNameString.Length / 2)..], endHeaders: true, handler: _handler);

Assert.Single(_handler.DecodedHeaders);
Assert.True(_handler.DecodedHeaders.ContainsKey(_literalHeaderNameString));
Assert.Equal(_headerValueString, _handler.DecodedHeaders[_literalHeaderNameString]);
}

[Fact]
public void DecodesLiteralHeaderFieldNeverIndexed_NewName_NameAndValueBrokenIntoSeparateBuffers()
{
byte[] encoded = _literalHeaderFieldWithoutIndexingNewName
.Concat(_literalHeaderName)
.Concat(_headerValue)
.ToArray();

_decoder.Decode(encoded[..^_headerValue.Length], endHeaders: false, handler: _handler);
_decoder.Decode(encoded[^_headerValue.Length..], endHeaders: true, handler: _handler);

Assert.Single(_handler.DecodedHeaders);
Assert.True(_handler.DecodedHeaders.ContainsKey(_literalHeaderNameString));
Assert.Equal(_headerValueString, _handler.DecodedHeaders[_literalHeaderNameString]);
}

[Fact]
public void DecodesLiteralHeaderFieldNeverIndexed_NewName_ValueLengthBrokenIntoSeparateBuffers()
{
byte[] encoded = _literalHeaderFieldWithoutIndexingNewName
.Concat(_literalHeaderName)
.Concat(_headerValue)
.ToArray();

_decoder.Decode(encoded[..^(_headerValue.Length - 1)], endHeaders: false, handler: _handler);
_decoder.Decode(encoded[^(_headerValue.Length - 1)..], endHeaders: true, handler: _handler);

Assert.Single(_handler.DecodedHeaders);
Assert.True(_handler.DecodedHeaders.ContainsKey(_literalHeaderNameString));
Assert.Equal(_headerValueString, _handler.DecodedHeaders[_literalHeaderNameString]);
}

[Fact]
public void DecodesLiteralHeaderFieldNeverIndexed_NewName_ValueBrokenIntoSeparateBuffers()
{
byte[] encoded = _literalHeaderFieldWithoutIndexingNewName
.Concat(_literalHeaderName)
.Concat(_headerValue)
.ToArray();

_decoder.Decode(encoded[..^(_headerValueString.Length / 2)], endHeaders: false, handler: _handler);
_decoder.Decode(encoded[^(_headerValueString.Length / 2)..], endHeaders: true, handler: _handler);

Assert.Single(_handler.DecodedHeaders);
Assert.True(_handler.DecodedHeaders.ContainsKey(_literalHeaderNameString));
Assert.Equal(_headerValueString, _handler.DecodedHeaders[_literalHeaderNameString]);
}

[Fact]
public void DecodesDynamicTableSizeUpdate()
{
Expand Down Expand Up @@ -500,10 +606,8 @@ public void DecodesStringLength_ExceedsLimit_Throws()
string string8191 = new string('a', MaxHeaderFieldSize - 1);
string string8193 = new string('a', MaxHeaderFieldSize + 1);
string string8194 = new string('a', MaxHeaderFieldSize + 2);

var bytes = new byte[3];
var success = IntegerEncoder.Encode(8194, 7, bytes, out var written);

byte[] encoded = _literalHeaderFieldWithoutIndexingNewName
.Concat(new byte[] { 0x7f, 0x80, 0x3f }) // 8191 encoded with 7-bit prefix, no Huffman encoding
.Concat(Encoding.ASCII.GetBytes(string8191))
Expand All @@ -520,14 +624,12 @@ public void DecodesStringLength_ExceedsLimit_Throws()
.Concat(new byte[] { 0x7f, 0x83, 0x3f }) // 8194 encoded with 7-bit prefix, no Huffman encoding
.Concat(Encoding.ASCII.GetBytes(string8194))
.ToArray();

var ex = Assert.Throws<HPackDecodingException>(() => decoder.Decode(encoded, endHeaders: true, handler: _handler));
Assert.Equal(SR.Format(SR.net_http_headers_exceeded_length, MaxHeaderFieldSize + 1), ex.Message);
Assert.Equal(string8191, _handler.DecodedHeaders[string8191]);
Assert.Equal(string8193, _handler.DecodedHeaders[string8193]);
Assert.False(_handler.DecodedHeaders.ContainsKey(string8194));
}

[Fact]
public void DecodesStringLength_IndividualBytes()
{
Expand Down
105 changes: 100 additions & 5 deletions src/Shared/test/Shared.Tests/runtime/Http3/QPackDecoderTest.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,11 +25,11 @@ public class QPackDecoderTests
// 4.5.4 - Literal Header Field With Name Reference - Static Table - Index 44 (content-type)
private static readonly byte[] _literalHeaderFieldWithNameReferenceStatic = new byte[] { 0x5f, 0x1d };

// 4.5.6 - Literal Field Line With Literal Name - (translate)
private static readonly byte[] _literalFieldLineWithLiteralName = new byte[] { 0x37, 0x02, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x6c, 0x61, 0x74, 0x65 };
// 4.5.6 - Literal Field Line With Literal Name - (literal-header-field)
private static readonly byte[] _literalFieldLineWithLiteralName = new byte[] { 0x37, 0x0d, 0x6c, 0x69, 0x74, 0x65, 0x72, 0x61, 0x6c, 0x2d, 0x68, 0x65, 0x61, 0x64, 0x65, 0x72, 0x2d, 0x66, 0x69, 0x65, 0x6c, 0x64 };

private const string _contentTypeString = "content-type";
private const string _translateString = "translate";
private const string _literalHeaderFieldString = "literal-header-field";

// n e w - h e a d e r *
// 10101000 10111110 00010110 10011100 10100011 10010000 10110110 01111111
Expand Down Expand Up @@ -97,7 +97,7 @@ public void DecodesLiteralFieldLineWithLiteralName_Value()
.Concat(_headerValue)
.ToArray();

TestDecodeWithoutIndexing(encoded, _translateString, _headerValueString);
TestDecodeWithoutIndexing(encoded, _literalHeaderFieldString, _headerValueString);
}

[Fact]
Expand Down Expand Up @@ -140,7 +140,7 @@ public void DecodesLiteralFieldLineWithLiteralName_HuffmanEncodedValue()
.Concat(_headerValueHuffman)
.ToArray();

TestDecodeWithoutIndexing(encoded, _translateString, _headerValueString);
TestDecodeWithoutIndexing(encoded, _literalHeaderFieldString, _headerValueString);
}

[Fact]
Expand Down Expand Up @@ -173,6 +173,101 @@ public void DecodesLiteralFieldLineWithLiteralName_LargeValues()
});
}

[Fact]
public void LiteralFieldWithoutNameReference_SingleBuffer()
{
byte[] encoded = _literalFieldLineWithLiteralName
.Concat(_headerValue)
.ToArray();

_decoder.Decode(new byte[] { 0x00, 0x00 }, endHeaders: false, handler: _handler);
_decoder.Decode(encoded, endHeaders: true, handler: _handler);

Assert.Single(_handler.DecodedHeaders);
Assert.True(_handler.DecodedHeaders.ContainsKey(_literalHeaderFieldString));
Assert.Equal(_headerValueString, _handler.DecodedHeaders[_literalHeaderFieldString]);
}

[Fact]
public void LiteralFieldWithoutNameReference_NameLengthBrokenIntoSeparateBuffers()
{
byte[] encoded = _literalFieldLineWithLiteralName
.Concat(_headerValue)
.ToArray();

_decoder.Decode(new byte[] { 0x00, 0x00 }, endHeaders: false, handler: _handler);
_decoder.Decode(encoded[..1], endHeaders: false, handler: _handler);
_decoder.Decode(encoded[1..], endHeaders: true, handler: _handler);

Assert.Single(_handler.DecodedHeaders);
Assert.True(_handler.DecodedHeaders.ContainsKey(_literalHeaderFieldString));
Assert.Equal(_headerValueString, _handler.DecodedHeaders[_literalHeaderFieldString]);
}

[Fact]
public void LiteralFieldWithoutNameReference_NameBrokenIntoSeparateBuffers()
{
byte[] encoded = _literalFieldLineWithLiteralName
.Concat(_headerValue)
.ToArray();

_decoder.Decode(new byte[] { 0x00, 0x00 }, endHeaders: false, handler: _handler);
_decoder.Decode(encoded[..(_literalHeaderFieldString.Length / 2)], endHeaders: false, handler: _handler);
_decoder.Decode(encoded[(_literalHeaderFieldString.Length / 2)..], endHeaders: true, handler: _handler);

Assert.Single(_handler.DecodedHeaders);
Assert.True(_handler.DecodedHeaders.ContainsKey(_literalHeaderFieldString));
Assert.Equal(_headerValueString, _handler.DecodedHeaders[_literalHeaderFieldString]);
}

[Fact]
public void LiteralFieldWithoutNameReference_NameAndValueBrokenIntoSeparateBuffers()
{
byte[] encoded = _literalFieldLineWithLiteralName
.Concat(_headerValue)
.ToArray();

_decoder.Decode(new byte[] { 0x00, 0x00 }, endHeaders: false, handler: _handler);
_decoder.Decode(encoded[..^_headerValue.Length], endHeaders: false, handler: _handler);
_decoder.Decode(encoded[^_headerValue.Length..], endHeaders: true, handler: _handler);

Assert.Single(_handler.DecodedHeaders);
Assert.True(_handler.DecodedHeaders.ContainsKey(_literalHeaderFieldString));
Assert.Equal(_headerValueString, _handler.DecodedHeaders[_literalHeaderFieldString]);
}

[Fact]
public void LiteralFieldWithoutNameReference_ValueLengthBrokenIntoSeparateBuffers()
{
byte[] encoded = _literalFieldLineWithLiteralName
.Concat(_headerValue)
.ToArray();

_decoder.Decode(new byte[] { 0x00, 0x00 }, endHeaders: false, handler: _handler);
_decoder.Decode(encoded[..^(_headerValue.Length - 1)], endHeaders: false, handler: _handler);
_decoder.Decode(encoded[^(_headerValue.Length - 1)..], endHeaders: true, handler: _handler);

Assert.Single(_handler.DecodedHeaders);
Assert.True(_handler.DecodedHeaders.ContainsKey(_literalHeaderFieldString));
Assert.Equal(_headerValueString, _handler.DecodedHeaders[_literalHeaderFieldString]);
}

[Fact]
public void LiteralFieldWithoutNameReference_ValueBrokenIntoSeparateBuffers()
{
byte[] encoded = _literalFieldLineWithLiteralName
.Concat(_headerValue)
.ToArray();

_decoder.Decode(new byte[] { 0x00, 0x00 }, endHeaders: false, handler: _handler);
_decoder.Decode(encoded[..^(_headerValueString.Length / 2)], endHeaders: false, handler: _handler);
_decoder.Decode(encoded[^(_headerValueString.Length / 2)..], endHeaders: true, handler: _handler);

Assert.Single(_handler.DecodedHeaders);
Assert.True(_handler.DecodedHeaders.ContainsKey(_literalHeaderFieldString));
Assert.Equal(_headerValueString, _handler.DecodedHeaders[_literalHeaderFieldString]);
}

public static readonly TheoryData<byte[]> _incompleteHeaderBlockData = new TheoryData<byte[]>
{
// Incomplete header
Expand Down