HTTP request not handling chunked Transfer Encoding

[elliothatch]

Hello! I am using Espruino on ESP32 to make HTTPS requests. This is working fine for short responses, but it isn't correctly handling long responses with "Transfer-Encoding: chunked"; the chunk size is being emitted in the first "data" event along with the beginning of the response, and then the response is closed.

I see that this issue has been addressed in the past: #1458

I am building from source on the latest master with reduced jsVars because I was getting out of memory issues on the precompiled firmware. Commit: 9a9d8ec

Sample code:

function getWeatherForecast(office, gridX, gridY) {
	var path = `/gridpoints/${office}/${gridX},${gridY}`;
	var options = {
		host: 'api.weather.gov',
		path: path,
		method: 'GET',
		protocol: 'https:',
		headers: {
			'User-Agent': 'Espruino',
			'Accept': 'application/geo+json'
		}
	};
	console.log(`GET https://api.weather.gov${path}`);
	var body = '';

	var req = require("http").request(options, function(res) {
		res.on('data', function(data) {
			body += data;
		});
		res.on('close', function() {
			console.log({
				code: res.statusCode,
				headers: res.headers,
		        });
		        console.log(body);
		});

		req.on('error', function(err) {
			console.log(`HTTP error ${err.code}: ${err.message}`);
			console.log(err.stack);
		});
	});

	req.end();
}

var ssid = 'xxx';
var password = 'xxx';

var wifi = require('Wifi');
wifi.connect(ssid, {password: password}, function(err) {
	if(err) {
		console.log(`Failed to connect to wifi: ${err.message}`);
		console.log(err);
		return;
	}

	console.log(`Connected to Wifi. IP address is: ${wifi.getIP().ip}`);
	getWeatherForecast('SLC', 100, 172);

});

Output:

Connected to Wifi. IP address is: 192.168.0.82
GET https://api.weather.gov/gridpoints/SLC/100,172
{
  "code": "200",
  "headers": {
    "Server": "nginx/1.20.1",
    "Content-Type": "application/geo+json",
    "Access-Control-Allow-Origin": "*",
    "Access-Control-Expose-Headers": "X-Correlation-Id, X-Request-Id, X-Server-Id",
    "X-Request-ID": "e56cc833-0ec2-463c-848c-7d7a67a48435",
    "X-Correlation-ID": "2d5e0f1",
    "X-Server-ID": "vm-bldr-nids-apiapp5.ncep.noaa.gov",
    "Cache-Control": "public, max-age=1653, s-maxage=3600",
    "Expires": "Fri, 24 Feb 2023 03:20:01 GMT",
    "Date": "Fri, 24 Feb 2023 02:52:28 GMT",
    "Transfer-Encoding": " chunked",
    "Connection": "close",
    "Connection": "Transfer-Encoding",
    "X-Edge-Request-ID": "3e2f508",
    "Vary": "Accept,Feature-Flags,Accept-Language",
    "Strict-Transport-Security": "max-age=31536000 ; includeSubDomains ; preload"
   }
 }
00006000
{
    "@context": [
        "https://geojson.org/geojson-ld/geojson-context.jsonld",
        {
            "@version": "1.1",
            "wmoUnit": "https://codes.wmo.int/common/unit/",
            "nwsUnit": "https://api.weather.gov/ontology/unit/"
        }
    ],
    "id": "https://api.weather.gov/gridpoints/SLC/100,172",
    "type": "Feature",
    "geometry": {
        "type": "Polygon",
        "coordinates": [
            [
                [
                    -111.8805053,
                    40.719880099999997
                ],
                [
                    -111.8769106,
                    40.698106799999998
                ],
                [
                    -111.8481231,
                    40.700835499999997
                ],
                [
                    -111.85171179999999,
                    40.7226091
                ],
                [
                    -111.8805053,

Thank you for your work on this project!

[gfwilliams]

The strange thing I'm seeing here is: "Transfer-Encoding": " chunked"

It seems like the headers include an extra space - so normally we'd have Transfer-Encoding:chunked but this server sends Transfer-Encoding: chunked

But looking at the spec that's allowed - I've just pushed a change to Espruino which should ignore whitespace after the ':' and before the value