Skip to content

Commit e6ed130

Browse files
Geod24Geod24
committed
feat: Implement support for Github-based index, bypassing the registry
This implement a package 'index' similar to that found for Homebrew, Nix, Cargo, etc... It allows us to remove a SPOF in our critical infrastructure, as a Github outage would always cause a registry being unusable anyway. There are multiple steps to having a useful index: - For transition purpose, we add a hidden command to Dub that export an `index.yaml`; - In the future, users should register their packages by adding an entry to `index.yaml`, the index definition file of the registry. This is used as the source of all packages; - `dub` now has a hidden `index-build` command to allow it to build the index based on an index definition file (`index.yaml`). Using this, it queries the various APIs to generate JSON index files that are stored under a pre-defined hierarchy. - Finally, a `PackageSupplier` is added to make use of this new feature; In the future, the registration process needs to be moved from the registry to Github to make this migration complete. This *can* be done by exposing a user-friendly interface on `code.dlang.org`, if making an MR to the index is deemed too complicated.
1 parent ec7b9f0 commit e6ed130

File tree

13 files changed

+2079
-3
lines changed

13 files changed

+2079
-3
lines changed

‎build-files.txt‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,12 @@ source/dub/generators/sublimetext.d
1818
source/dub/generators/targetdescription.d
1919
source/dub/generators/visuald.d
2020
source/dub/init.d
21+
source/dub/index/bitbucket.d
22+
source/dub/index/client.d
23+
source/dub/index/data.d
24+
source/dub/index/github.d
25+
source/dub/index/gitlab.d
26+
source/dub/index/utils.d
2127
source/dub/internal/configy/attributes.d
2228
source/dub/internal/configy/backend/node.d
2329
source/dub/internal/configy/backend/yaml.d

‎source/dub/commandline.d‎

Lines changed: 255 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -73,7 +73,10 @@ CommandGroup[] getCommands() @safe pure nothrow
7373
new ListOverridesCommand,
7474
new CleanCachesCommand,
7575
new ConvertCommand,
76-
)
76+
// This is index management but those commands are hidden
77+
new IndexBuildCommand,
78+
new IndexFromRegistryCommand,
79+
),
7780
];
7881
}
7982

@@ -273,7 +276,8 @@ unittest {
273276
assert(handler.commandNames == ["init", "run", "build", "test", "lint", "generate",
274277
"describe", "clean", "dustmite", "fetch", "add", "remove",
275278
"upgrade", "add-path", "remove-path", "add-local", "remove-local", "list", "search",
276-
"add-override", "remove-override", "list-overrides", "clean-caches", "convert"]);
279+
"add-override", "remove-override", "list-overrides", "clean-caches", "convert",
280+
"index-build", "index-fromregistry"]);
277281
}
278282

279283
/// It sets the cwd as root_path by default
@@ -2983,6 +2987,255 @@ class ConvertCommand : Command {
29832987
}
29842988

29852989

2990+
/******************************************************************************/
2991+
/* Index management
2992+
/******************************************************************************/
2993+
2994+
public class IndexBuildCommand : Command {
2995+
import dub.index.bitbucket;
2996+
import dub.index.data;
2997+
import dub.index.github;
2998+
import dub.index.gitlab;
2999+
import dub.index.utils;
3000+
import std.random;
3001+
import std.range;
3002+
3003+
/// Index file to use
3004+
private string index = "index.yaml";
3005+
/// Filename to write to
3006+
private string output = "index-build-result";
3007+
/// Packages to filter in - assume all if empty
3008+
private string[] include;
3009+
/// Packages to filter out
3010+
private string[] exclude;
3011+
/// Bearer token to use to authenticate requests
3012+
private string githubToken, gitlabToken, bitbucketToken;
3013+
/// Kind of packages to include (default: all kinds)
3014+
private string[] kind;
3015+
/// Whether to force the iteration of tags or not
3016+
/// This needs to be used if some tags need to be reprocessed
3017+
private bool force_tags;
3018+
/// Force the package to be entirely reprocessed. Imply `--force-tags`.
3019+
private bool force;
3020+
/// Whether to use a randomized sample of packages
3021+
private bool random;
3022+
/// The number of packages to update
3023+
private uint maxUpdates = uint.max;
3024+
/// Source and target index to use, mutually exclusive with `random`
3025+
private uint fromIdx = 0, toIdx = uint.max;
3026+
3027+
this() @safe pure nothrow
3028+
{
3029+
this.name = "index-build";
3030+
this.description = "Generate the rich index from the index.yaml file";
3031+
this.helpText = [ "This command is for internal use only. Do not use it." ];
3032+
this.hidden = true;
3033+
}
3034+
3035+
override void prepare(scope CommandArgs args) {
3036+
args.getopt("bitbucket-token", &this.bitbucketToken, ["Bearer token to use when issuing Bitbucket requests"]);
3037+
args.getopt("github-token", &this.githubToken, ["Bearer token to use when issuing Github requests"]);
3038+
args.getopt("gitlab-token", &this.gitlabToken, ["Bearer token to use when issuing GitLab requests"]);
3039+
args.getopt("output", &this.output, ["Where to output the data (path to a folder)"]);
3040+
args.getopt("index", &this.index, ["Index file to use - default to 'index.yaml'"]);
3041+
args.getopt("include", &this.include, ["Which packages to filter in - if not, assume all"]);
3042+
args.getopt("exclude", &this.exclude, ["Which packages to filter out - if not, assume none"]);
3043+
args.getopt("kind", &this.kind, ["Kind of packages to include (github, gitlab, bitbucket). Default: all"]);
3044+
args.getopt("force", &this.force, ["Force Dub to reprocess packages even if it has cache informations"]);
3045+
args.getopt("force-tags", &this.force_tags,
3046+
["Force Dub to re-list tags, but do not reload the recipe if the commit hasn't changed."]);
3047+
args.getopt("random", &this.random, ["Randomize the order in which packages are processed"]);
3048+
args.getopt("max-updates", &this.maxUpdates, ["Maximum number of packages to process"]);
3049+
args.getopt("from", &this.fromIdx, ["Index to seek to before iterating the list of packages (default: 0)"]);
3050+
args.getopt("to", &this.toIdx, ["Index to stop at when iterating the list of packages (default: end of list)"]);
3051+
3052+
enforce(this.fromIdx <= this.toIdx, "Cannot have source index (`--from`) be past end index (`--to`)");
3053+
enforce(this.fromIdx == 0 || !this.random,
3054+
"Cannot specify source index (`--from`) for random sampling (`--random`)");
3055+
enforce(this.toIdx == uint.max || !this.random,
3056+
"Cannot specify end index (`--to`) for random sampling (`--random`)");
3057+
}
3058+
3059+
override int execute(Dub dub, string[] free_args, string[] app_args)
3060+
{
3061+
import dub.index.client : RepositoryClient;
3062+
import dub.index.data;
3063+
import dub.internal.configy.easy;
3064+
static import std.file;
3065+
import std.typecons;
3066+
3067+
enforceUsage(free_args.length == 0, "Expected no free argument.");
3068+
enforceUsage(app_args.length == 0, "Expected zero application arguments.");
3069+
3070+
const isUpdate = std.file.exists(this.output);
3071+
if (isUpdate)
3072+
enforce(std.file.isDir(this.output), this.output ~ ": is not a directory");
3073+
else
3074+
std.file.mkdirRecurse(this.output);
3075+
3076+
auto indexN = parseConfigFileSimple!PackageList(this.index);
3077+
if (indexN.isNull()) return 1;
3078+
auto indexC = indexN.get();
3079+
logInfoNoTag("Found %s packages in the index file", indexC.packages.length);
3080+
3081+
const NativePath outputPath = NativePath(this.output);
3082+
size_t processed, updated, notsupported;
3083+
string[] included, excluded, errored;
3084+
scope gh = new GithubClient(this.githubToken);
3085+
scope gl = new GitLabClient(this.gitlabToken);
3086+
scope bb = new BitbucketClient(this.bitbucketToken);
3087+
3088+
void update (scope RepositoryClient client, in PackageEntry pkg) {
3089+
const target = getPackageDescriptionPath(outputPath, PackageName(pkg.name.value));
3090+
ensureDirectory(target.parentPath());
3091+
const targetStr = target.toNativeString();
3092+
auto previous = !this.force && std.file.exists(targetStr) ?
3093+
parseConfigFileSimple!(IndexedPackage!0)(targetStr, StrictMode.Ignore) :
3094+
Nullable!(IndexedPackage!0).init;
3095+
if (this.force_tags && !previous.isNull())
3096+
previous.get().cache = CacheInfo.init;
3097+
auto res = updateDescription(client, pkg, previous);
3098+
if (previous.isNull() || previous.get() != res) {
3099+
std.file.write(targetStr, res.serializeToJsonString());
3100+
++updated;
3101+
}
3102+
}
3103+
3104+
// Update a single package - this code is in its own function as it
3105+
// is wrapped in a try-catch in the `foreach` to process as many packages
3106+
// as possible
3107+
void updatePackageIndex (in PackageEntry pkg) {
3108+
logInfo("[%s] Processing included package", pkg.name.value);
3109+
switch (pkg.source.kind) {
3110+
case `github`:
3111+
scope client = gh.new Repository(pkg.source.owner, pkg.source.project);
3112+
update(client, pkg);
3113+
break;
3114+
case `gitlab`:
3115+
scope client = gl.new Project(pkg.source.owner, pkg.source.project);
3116+
update(client, pkg);
3117+
break;
3118+
case `bitbucket`:
3119+
scope client = bb.new Repository(pkg.source.owner, pkg.source.project);
3120+
update(client, pkg);
3121+
break;
3122+
default:
3123+
throw new Exception("Package kind not supported: " ~ pkg.source.kind);
3124+
}
3125+
}
3126+
3127+
int processEntry (size_t idx, ref PackageEntry pkg) {
3128+
if (updated >= this.maxUpdates) return 1;
3129+
if (this.include.length && !this.include.canFind(pkg.name.value))
3130+
return 0;
3131+
if (this.exclude.canFind(pkg.name.value)) {
3132+
excluded ~= pkg.name.value;
3133+
return 0;
3134+
}
3135+
if (this.kind.length && !this.kind.canFind(pkg.source.kind))
3136+
return 0;
3137+
3138+
++processed;
3139+
try
3140+
updatePackageIndex(pkg);
3141+
catch (Exception exc) {
3142+
errored ~= pkg.name.value;
3143+
// If we get a 404 here, it might be a dead package
3144+
logError("[%s] Could not build index for package: %s",
3145+
pkg.name.value, exc.message());
3146+
}
3147+
3148+
if (this.include.length) {
3149+
included ~= pkg.name.value;
3150+
if (included.length % 10 == 0) {
3151+
const rl = gh.getRateLimit();
3152+
logDebug("Requests still available: %s/%s", rl.remaining, rl.limit);
3153+
}
3154+
}
3155+
else if (idx % 10 == 0) {
3156+
const rl = gh.getRateLimit();
3157+
logDebug("Requests still available: %s/%s", rl.remaining, rl.limit);
3158+
}
3159+
return 0;
3160+
}
3161+
3162+
if (this.random) { // Can't use `std.random : choose` because bugs
3163+
foreach (idx, pkg; indexC.packages.randomCover().enumerate)
3164+
if (processEntry(idx, pkg))
3165+
break;
3166+
} else {
3167+
const startIdx = min(this.fromIdx, indexC.packages.length);
3168+
const endIdx = min(this.toIdx, indexC.packages.length);
3169+
foreach (idx, pkg; indexC.packages[startIdx .. endIdx])
3170+
if (processEntry(idx, pkg))
3171+
break;
3172+
}
3173+
3174+
logInfoNoTag("Updated %s packages out of %s processed (%s excluded, %s errors, %s not supported)",
3175+
updated, processed, excluded.length, errored.length, notsupported);
3176+
if (this.include.length && included != this.include)
3177+
logWarn("Not all explicitly-included packages have been processed!");
3178+
if (errored.length)
3179+
logWarn("The following packages errored out:\n%(\t- %s\n%)", errored);
3180+
if (!this.kind.length || this.kind.canFind(`github`)) {
3181+
const rl = gh.getRateLimit();
3182+
logInfoNoTag("Github requests still available: %s/%s", rl.remaining, rl.limit);
3183+
}
3184+
return 0;
3185+
}
3186+
}
3187+
3188+
public class IndexFromRegistryCommand : Command {
3189+
/// Filename to write to
3190+
private string output = "index.yaml";
3191+
/// Bypass cache, always query the registry
3192+
private bool force;
3193+
3194+
this() @safe pure nothrow
3195+
{
3196+
this.name = "index-fromregistry";
3197+
this.description = "Generate the index.yaml file from the remote registry";
3198+
this.helpText = [ "This command is for internal use only. Do not use it." ];
3199+
this.hidden = true;
3200+
}
3201+
3202+
override void prepare(scope CommandArgs args) {
3203+
args.getopt("O", &this.output, ["Where to output the data ('-' is supported)"]);
3204+
args.getopt("f", &this.force, ["Bypass the cache and always query the registry"]);
3205+
}
3206+
3207+
override int execute(Dub dub, string[] free_args, string[] app_args)
3208+
{
3209+
import dub.internal.vibecompat.inet.url;
3210+
import dub.packagesuppliers.registry;
3211+
import std.format;
3212+
3213+
enforceUsage(free_args.length == 0, "Expected zero arguments.");
3214+
enforceUsage(app_args.length == 0, "Expected zero application arguments.");
3215+
3216+
scope registry = new RegistryPackageSupplier(URL(defaultRegistryURLs[1]));
3217+
scope allPkgs = registry.getPackageDump(this.force);
3218+
writeln("Found ", allPkgs.array.length, " packages");
3219+
scope output = this.output == "-" ? stdout : File(this.output, "w+");
3220+
scope writer = output.lockingTextWriter();
3221+
writer.formattedWrite("packages:\n");
3222+
foreach (pkg; allPkgs.array) {
3223+
writer.formattedWrite(` %s:
3224+
source:
3225+
kind: %s
3226+
owner: %s
3227+
project: %s
3228+
`,
3229+
pkg["name"].opt!string, pkg["repository"]["kind"].opt!string,
3230+
pkg["repository"]["owner"].opt!string,
3231+
pkg["repository"]["project"].opt!string);
3232+
}
3233+
3234+
return 0;
3235+
}
3236+
}
3237+
3238+
29863239
/******************************************************************************/
29873240
/* HELP */
29883241
/******************************************************************************/

‎source/dub/dub.d‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,7 @@ deprecated("use defaultRegistryURLs") enum defaultRegistryURL = defaultRegistryU
4747

4848
/// The URL to the official package registry and it's default fallback registries.
4949
static immutable string[] defaultRegistryURLs = [
50+
"dub+index+https://github.com/dlang/dub-index.git",
5051
"https://code.dlang.org/",
5152
"https://codemirror.dlang.org/"
5253
];
@@ -413,6 +414,12 @@ class Dub {
413414
switch (url.startsWith("dub+", "mvn+", "file://"))
414415
{
415416
case 1:
417+
// `startsWith` takes the shortest match so if we provide `dub+`
418+
// `and `dub+index+` it will always match the former...
419+
if (url.startsWith(`dub+index+`)) {
420+
return new IndexPackageSupplier(URL(url["dub+index+".length .. $]),
421+
this.m_dirs.userPackages ~ "index");
422+
}
416423
return new RegistryPackageSupplier(URL(url[4..$]));
417424
case 2:
418425
return new MavenRegistryPackageSupplier(URL(url[4..$]));
@@ -1291,7 +1298,7 @@ class Dub {
12911298
try
12921299
results ~= tuple(ps.description, ps.searchPackages(query));
12931300
catch (Exception e) {
1294-
logWarn("Searching %s for '%s' failed: %s", ps.description, query, e.msg);
1301+
logWarn("Searching %s for '%s' failed: %s", ps.description, query, e);
12951302
}
12961303
}
12971304
return results.filter!(tup => tup[1].length);

0 commit comments

Comments
 (0)