Skip to content

Commit

Permalink
Support fetching dependencies over git+http(s)
Browse files Browse the repository at this point in the history 
Closes #14298

This commit adds support for fetching dependencies over git+http(s)
using a minimal implementation of the Git protocols and formats relevant
to fetching repository data.

Git URLs can be specified in `build.zig.zon` as follows:

```zig
.xml = .{
    .url = "git+https://github.com/ianprime0509/zig-xml#7380d59d50f1cd8460fd748b5f6f179306679e2f",
    .hash = "122085c1e4045fa9cb69632ff771c56acdb6760f34ca5177e80f70b0b92cd80da3e9",
},
```

The fragment part of the URL may specify a commit ID (SHA1 hash), branch
name, or tag. It is an error to omit the fragment: if this happens, the
compiler will prompt the user to add it, using the commit ID of the HEAD
commit of the repository (that is, the latest commit of the default
branch):

```
Fetch Packages... xml... /var/home/ian/src/zig-gobject/build.zig.zon:6:20: error: url field is missing an explicit ref
            .url = "git+https://github.com/ianprime0509/zig-xml",
                   ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
note: try .url = "git+https://github.com/ianprime0509/zig-xml#dfdc044f3271641c7d428dc8ec8cd46423d8b8b6",
```

This implementation currently supports only version 2 of Git's wire
protocol (documented in
[protocol-v2](https://git-scm.com/docs/protocol-v2)), which was first
introduced in Git 2.19 (2018) and made the default in 2.26 (2020).

The wire protocol behaves similarly when used over other transports,
such as SSH and the "Git protocol" (git:// URLs), so it should be
reasonably straightforward to support fetching dependencies from such
URLs if the necessary transports are implemented (e.g. #14295).
  • Loading branch information
@ianprime0509
ianprime0509 committed Sep 29, 2023
1 parent acac685 commit 52a0f49
Show file tree
Hide file tree
Showing 4 changed files with 1,619 additions and 58 deletions.
256 changes: 198 additions & 58 deletions src/Package.zig
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ const Module = @import("Module.zig");
const Cache = std.Build.Cache;
const build_options = @import("build_options");
const Manifest = @import("Manifest.zig");
const git = @import("git.zig");

pub const Table = std.StringHashMapUnmanaged(*Package);

Expand Down Expand Up @@ -647,65 +648,12 @@ fn fetchAndUnpack(
};
defer tmp_directory.closeAndFree(gpa);

var h = std.http.Headers{ .allocator = gpa };
defer h.deinit();

var req = try http_client.request(.GET, uri, h, .{});
defer req.deinit();

try req.start(.{});
try req.wait();

if (req.response.status != .ok) {
return report.fail(dep.url_tok, "Expected response status '200 OK' got '{} {s}'", .{
@intFromEnum(req.response.status),
req.response.status.phrase() orelse "",
});
}

const content_type = req.response.headers.getFirstValue("Content-Type") orelse
return report.fail(dep.url_tok, "Missing 'Content-Type' header", .{});

var prog_reader: ProgressReader(std.http.Client.Request.Reader) = .{
.child_reader = req.reader(),
.prog_node = &pkg_prog_node,
.unit = if (req.response.content_length) |content_length| unit: {
const kib = content_length / 1024;
const mib = kib / 1024;
if (mib > 0) {
pkg_prog_node.setEstimatedTotalItems(@intCast(mib));
pkg_prog_node.setUnit("MiB");
break :unit .mib;
} else {
pkg_prog_node.setEstimatedTotalItems(@intCast(@max(1, kib)));
pkg_prog_node.setUnit("KiB");
break :unit .kib;
}
} else .any,
};
pkg_prog_node.context.refresh();

if (ascii.eqlIgnoreCase(content_type, "application/gzip") or
ascii.eqlIgnoreCase(content_type, "application/x-gzip") or
ascii.eqlIgnoreCase(content_type, "application/tar+gzip"))
{
// I observed the gzip stream to read 1 byte at a time, so I am using a
// buffered reader on the front of it.
try unpackTarball(gpa, prog_reader.reader(), tmp_directory.handle, std.compress.gzip);
} else if (ascii.eqlIgnoreCase(content_type, "application/x-xz")) {
// I have not checked what buffer sizes the xz decompression implementation uses
// by default, so the same logic applies for buffering the reader as for gzip.
try unpackTarball(gpa, prog_reader.reader(), tmp_directory.handle, std.compress.xz);
} else if (ascii.eqlIgnoreCase(content_type, "application/octet-stream")) {
// support gitlab tarball urls such as https://gitlab.com/<namespace>/<project>/-/archive/<sha>/<project>-<sha>.tar.gz
// whose content-disposition header is: 'attachment; filename="<project>-<sha>.tar.gz"'
const content_disposition = req.response.headers.getFirstValue("Content-Disposition") orelse
return report.fail(dep.url_tok, "Missing 'Content-Disposition' header for Content-Type=application/octet-stream", .{});
if (isTarAttachment(content_disposition)) {
try unpackTarball(gpa, prog_reader.reader(), tmp_directory.handle, std.compress.gzip);
} else return report.fail(dep.url_tok, "Unsupported 'Content-Disposition' header value: '{s}' for Content-Type=application/octet-stream", .{content_disposition});
if (mem.eql(u8, uri.scheme, "http") or mem.eql(u8, uri.scheme, "https")) {
try fetchHttp(http_client, tmp_directory, dep, uri, report, &pkg_prog_node);
} else if (mem.eql(u8, uri.scheme, "git+http") or mem.eql(u8, uri.scheme, "git+https")) {
try fetchGit(http_client, tmp_directory, dep, uri, report, &pkg_prog_node);
} else {
return report.fail(dep.url_tok, "Unsupported 'Content-Type' header value: '{s}'", .{content_type});
return report.fail(dep.url_tok, "unsupported URL scheme: {s}", .{uri.scheme});
}

// Download completed - stop showing downloaded amount as progress
Expand Down Expand Up @@ -776,6 +724,198 @@ fn fetchAndUnpack(
};
}

fn fetchHttp(
http_client: *std.http.Client,
tmp_directory: Compilation.Directory,
dep: Manifest.Dependency,
uri: std.Uri,
report: Report,
pkg_prog_node: *std.Progress.Node,
) !void {
const gpa = http_client.allocator;

var h = std.http.Headers{ .allocator = gpa };
defer h.deinit();

var req = try http_client.request(.GET, uri, h, .{});
defer req.deinit();

try req.start(.{});
try req.wait();

if (req.response.status != .ok) {
return report.fail(dep.url_tok, "Expected response status '200 OK' got '{} {s}'", .{
@intFromEnum(req.response.status),
req.response.status.phrase() orelse "",
});
}

const content_type = req.response.headers.getFirstValue("Content-Type") orelse
return report.fail(dep.url_tok, "Missing 'Content-Type' header", .{});

var prog_reader: ProgressReader(std.http.Client.Request.Reader) = .{
.child_reader = req.reader(),
.prog_node = pkg_prog_node,
.unit = if (req.response.content_length) |content_length| unit: {
const kib = content_length / 1024;
const mib = kib / 1024;
if (mib > 0) {
pkg_prog_node.setEstimatedTotalItems(@intCast(mib));
pkg_prog_node.setUnit("MiB");
break :unit .mib;
} else {
pkg_prog_node.setEstimatedTotalItems(@intCast(@max(1, kib)));
pkg_prog_node.setUnit("KiB");
break :unit .kib;
}
} else .any,
};
pkg_prog_node.context.refresh();

if (ascii.eqlIgnoreCase(content_type, "application/gzip") or
ascii.eqlIgnoreCase(content_type, "application/x-gzip") or
ascii.eqlIgnoreCase(content_type, "application/tar+gzip"))
{
// I observed the gzip stream to read 1 byte at a time, so I am using a
// buffered reader on the front of it.
try unpackTarball(gpa, prog_reader.reader(), tmp_directory.handle, std.compress.gzip);
} else if (ascii.eqlIgnoreCase(content_type, "application/x-xz")) {
// I have not checked what buffer sizes the xz decompression implementation uses
// by default, so the same logic applies for buffering the reader as for gzip.
try unpackTarball(gpa, prog_reader.reader(), tmp_directory.handle, std.compress.xz);
} else if (ascii.eqlIgnoreCase(content_type, "application/octet-stream")) {
// support gitlab tarball urls such as https://gitlab.com/<namespace>/<project>/-/archive/<sha>/<project>-<sha>.tar.gz
// whose content-disposition header is: 'attachment; filename="<project>-<sha>.tar.gz"'
const content_disposition = req.response.headers.getFirstValue("Content-Disposition") orelse
return report.fail(dep.url_tok, "Missing 'Content-Disposition' header for Content-Type=application/octet-stream", .{});
if (isTarAttachment(content_disposition)) {
try unpackTarball(gpa, prog_reader.reader(), tmp_directory.handle, std.compress.gzip);
} else return report.fail(dep.url_tok, "Unsupported 'Content-Disposition' header value: '{s}' for Content-Type=application/octet-stream", .{content_disposition});
} else {
return report.fail(dep.url_tok, "Unsupported 'Content-Type' header value: '{s}'", .{content_type});
}
}

fn fetchGit(
http_client: *std.http.Client,
tmp_directory: Compilation.Directory,
dep: Manifest.Dependency,
uri: std.Uri,
report: Report,
pkg_prog_node: *std.Progress.Node,
) !void {
const gpa = http_client.allocator;
var transport_uri = uri;
transport_uri.scheme = uri.scheme["git+".len..];
var redirect_uri: []u8 = undefined;
var session: git.Session = .{ .transport = http_client, .uri = transport_uri };
session.discoverCapabilities(gpa, &redirect_uri) catch |e| switch (e) {
error.Redirected => {
defer gpa.free(redirect_uri);
return report.fail(dep.url_tok, "repository moved to {s}", .{redirect_uri});
},
else => |other| return other,
};

const want_oid = want_oid: {
const want_ref = uri.fragment orelse "HEAD";
if (git.parseOid(want_ref)) |oid| break :want_oid oid else |_| {}

const want_ref_head = try std.fmt.allocPrint(gpa, "refs/heads/{s}", .{want_ref});
defer gpa.free(want_ref_head);
const want_ref_tag = try std.fmt.allocPrint(gpa, "refs/tags/{s}", .{want_ref});
defer gpa.free(want_ref_tag);

var ref_iterator = try session.listRefs(gpa, .{
.ref_prefixes = &.{ want_ref, want_ref_head, want_ref_tag },
.include_peeled = true,
});
defer ref_iterator.deinit();
while (try ref_iterator.next()) |ref| {
if (mem.eql(u8, ref.name, want_ref) or
mem.eql(u8, ref.name, want_ref_head) or
mem.eql(u8, ref.name, want_ref_tag))
{
break :want_oid ref.peeled orelse ref.oid;
}
}
return report.fail(dep.url_tok, "ref not found: {s}", .{want_ref});
};
if (uri.fragment == null) {
const file_path = try report.directory.join(gpa, &.{Manifest.basename});
defer gpa.free(file_path);

const eb = report.error_bundle;
const notes_len = 1;
try Report.addErrorMessage(report.ast.*, file_path, eb, notes_len, .{
.tok = dep.url_tok,
.off = 0,
.msg = "url field is missing an explicit ref",
});
const notes_start = try eb.reserveNotes(notes_len);
eb.extra.items[notes_start] = @intFromEnum(try eb.addErrorMessage(.{
.msg = try eb.printString("try .url = \"{+/}#{}\",", .{ uri, std.fmt.fmtSliceHexLower(&want_oid) }),
}));
return error.PackageFetchFailed;
}

// The .git directory is used to store the packfile and associated index, but
// we do not attempt to replicate the exact structure of a real .git
// directory, since that isn't relevant for fetching a package.
{
var pack_dir = try tmp_directory.handle.makeOpenPath(".git", .{});
defer pack_dir.close();
var pack_file = try pack_dir.createFile("pkg.pack", .{ .read = true });
defer pack_file.close();
{
var want_oid_buf: [git.fmt_oid_length]u8 = undefined;
_ = std.fmt.bufPrint(&want_oid_buf, "{}", .{std.fmt.fmtSliceHexLower(&want_oid)}) catch unreachable;
var fetch_stream = try session.fetch(gpa, &.{&want_oid_buf});
defer fetch_stream.deinit();

var fetch_prog_node = pkg_prog_node.start("Fetch", 0);
defer fetch_prog_node.end();
fetch_prog_node.activate();
fetch_prog_node.context.refresh();
var prog_reader: ProgressReader(git.Session.FetchStream.Reader) = .{
.child_reader = fetch_stream.reader(),
.prog_node = pkg_prog_node,
.unit = .any,
};
pkg_prog_node.context.refresh();

var fifo = std.fifo.LinearFifo(u8, .{ .Static = 4096 }).init();
try fifo.pump(prog_reader.reader(), pack_file.writer());
try pack_file.sync();
}

var index_file = try pack_dir.createFile("pkg.idx", .{ .read = true });
defer index_file.close();
{
var index_prog_node = pkg_prog_node.start("Index pack", 0);
defer index_prog_node.end();
index_prog_node.activate();
index_prog_node.context.refresh();
var index_buffered_writer = std.io.bufferedWriter(index_file.writer());
try git.indexPack(gpa, pack_file, index_buffered_writer.writer());
try index_buffered_writer.flush();
try index_file.sync();
}

{
var checkout_prog_node = pkg_prog_node.start("Checkout", 0);
defer checkout_prog_node.end();
checkout_prog_node.activate();
checkout_prog_node.context.refresh();
var repository = try git.Repository.init(gpa, pack_file, index_file);
defer repository.deinit();
try repository.checkout(tmp_directory.handle, want_oid);
}
}

try tmp_directory.handle.deleteTree(".git");
}

fn unpackTarball(
gpa: Allocator,
req_reader: anytype,
Expand Down
Loading

0 comments on commit 52a0f49

Please sign in to comment.