refactor(download): refactor download module
test / test (push) Failing after 3m2s

This commit is contained in:
2026-10-27 01:02:12 +03:00
parent 188fcc8e4c
commit d91021982a
7 changed files with 105 additions and 79 deletions
+1 -1
View File
@@ -3,5 +3,5 @@ import { Preset } from "gulp-haxetool";
import presetData from "./haxetool.json" with { type: "json" }; import presetData from "./haxetool.json" with { type: "json" };
import packageInfo from "./package.json" with { type: "json" }; import packageInfo from "./package.json" with { type: "json" };
const preset = new Preset(presetData, null, packageInfo); const preset = new Preset(presetData, { UseRuffle: true }, packageInfo);
export default preset.register(gulp); export default preset.register(gulp);
+74 -52
View File
@@ -1,59 +1,26 @@
import progress from "cli-progress"; import progress from "cli-progress";
import fs from "fs"; import fs from "fs";
import fse from "fs-extra"; import fse from "fs-extra";
import got from "got"; import got, { type Progress, type Request } from "got";
import path from "path"; import path from "path";
import ps from "promise-streams"; import ps from "promise-streams";
import type { Stream, Writable } from "stream"; import type { Stream, Writable } from "stream";
import * as tar from "tar"; import { x as tarX } from "tar";
import unzip from "unzip-stream"; import unzip from "unzip-stream";
export class Downloader { export interface DownloadOptions {
static bar: progress.MultiBar | null; progress?: boolean;
strip?: number;
static createProgressBar() { executable?: boolean;
if (Downloader.bar == null) {
Downloader.bar = new progress.MultiBar(
{
clearOnComplete: true,
},
progress.Presets.shades_grey,
);
}
return Downloader.bar.create(0, 0);
} }
static releaseProgressBar(bar: progress.SingleBar) { const DEFAULT_OPTIONS: DownloadOptions = { progress: true, strip: 1, executable: false };
bar.stop();
if (Downloader.bar != null) {
Downloader.bar.remove(bar);
if (!Downloader.bar.isActive) {
Downloader.bar.stop();
Downloader.bar = null;
}
}
}
static download(url: string, dest: string, strip = 1, executable = false) { type Extractor = (stream: Request, dest: string, options: DownloadOptions) => Stream;
fse.ensureDirSync(dest);
const bar = Downloader.createProgressBar(); const ZIP_EXTRACTOR: Extractor = (stream: Request, dest: string, options: DownloadOptions): Stream => {
let stream: Stream = got.stream(url); return stream.pipe(unzip.Parse()).on("entry", (entry: any) => {
stream = stream const filePath = entry.path.split("/").slice(options.strip).join(path.sep);
.on("request", (r: any) => {
if (bar) bar.start(1, 0);
})
.on("downloadProgress", (p: any) => {
if (bar) {
bar.setTotal(p.total);
bar.update(p.transferred);
if (p.total === p.transferred) {
Downloader.releaseProgressBar(bar);
}
}
});
if (url.endsWith(".zip")) {
stream = stream.pipe(unzip.Parse()).on("entry", (entry: any) => {
const filePath = entry.path.split("/").slice(strip).join(path.sep);
if (filePath.length > 0) { if (filePath.length > 0) {
if (entry.type === "Directory") { if (entry.type === "Directory") {
fse.ensureDirSync(path.join(dest, path.normalize(filePath))); fse.ensureDirSync(path.join(dest, path.normalize(filePath)));
@@ -65,8 +32,8 @@ export class Downloader {
} }
entry.pipe( entry.pipe(
fs.createWriteStream(path.join(dest, path.normalize(filePath)), { fs.createWriteStream(path.join(dest, path.normalize(filePath)), {
// ToDo: zip entry file mode? // TODO: zip entry file mode?
mode: executable ? 0o755 : undefined, mode: options.executable ? 0o755 : undefined,
}), }),
); );
} }
@@ -74,11 +41,66 @@ export class Downloader {
entry.autodrain(); entry.autodrain();
} }
}); });
} else if (url.endsWith("tar.gz")) { };
stream = stream.pipe(tar.x({ C: dest, strip }) as unknown as Writable);
} else { const QZ_EXTRACTOR: Extractor = (stream: Request, dest: string, options: DownloadOptions): Stream => {
stream = stream.pipe(fs.createWriteStream(path.join(dest, url.split("/").pop()!))); // TODO: cast tar.Unpack to Writable
return stream.pipe(tarX({ C: dest, strip: options.strip }) as unknown as Writable);
};
const DEFAULT_EXTRACTOR: Extractor = (stream: Request, dest: string, options: DownloadOptions): Stream => {
const filename = String(stream.requestUrl as URL)
.split("/")
.pop()!;
return stream.pipe(fs.createWriteStream(path.join(dest, filename)));
};
const EXTRACTOR_MAP: Record<string, Extractor> = {
".zip": ZIP_EXTRACTOR,
".gz": QZ_EXTRACTOR,
"": DEFAULT_EXTRACTOR,
};
class ProgressHandler {
private multiBar: progress.MultiBar = new progress.MultiBar(
{
stopOnComplete: true,
},
progress.Presets.shades_classic,
);
handle(request: Request): Request {
const bar = this.multiBar.create(0, 0);
return request
.on("request", (request: Request) => {
bar.start(1, 0);
})
.on("downloadProgress", (progress: Progress) => {
if (progress.total) {
bar.setTotal(progress.total);
} }
return ps.wait(stream); bar.update(progress.transferred);
if (progress.total === progress.transferred) {
bar.render();
bar.stop();
this.multiBar.remove(bar);
}
});
} }
} }
const progressHandler = new ProgressHandler();
export const download = async (url: string, dest: string, options?: DownloadOptions): Promise<void> => {
options = Object.assign({}, DEFAULT_OPTIONS, options);
fse.ensureDirSync(dest);
let stream: Request = got.stream(url);
if (options.progress) {
stream = progressHandler.handle(stream);
}
const ext = path.extname(url);
const extractor = EXTRACTOR_MAP[ext] || EXTRACTOR_MAP[""];
return ps.wait(extractor(stream, dest, options));
};
export default download;
+3 -1
View File
@@ -52,4 +52,6 @@ export class OutputLogger extends Logger {
export const OUTPUT_LOGGER = new OutputLogger(); export const OUTPUT_LOGGER = new OutputLogger();
export default (line: string) => OUTPUT_LOGGER.log(line); export const output = (line: string) => OUTPUT_LOGGER.log(line);
export default output;
+2 -2
View File
@@ -1,6 +1,6 @@
import os from "os"; import os from "os";
import path from "path"; import path from "path";
import { Downloader } from "../common/download"; import download from "../common/download";
import { Logger } from "../common/log"; import { Logger } from "../common/log";
export class Sdk { export class Sdk {
@@ -55,7 +55,7 @@ export class Sdk {
return Promise.resolve(); return Promise.resolve();
} else { } else {
this.log.i("download: *%s*", this.link); this.log.i("download: *%s*", this.link);
return Downloader.download(this.link, this.path, strip); return download(this.link, this.path, { strip });
} }
} }
} }
+3 -3
View File
@@ -2,7 +2,7 @@ import fs from "fs";
import os from "os"; import os from "os";
import path from "path"; import path from "path";
import through from "through2"; import through from "through2";
import { Downloader } from "../common/download"; import download from "../common/download";
import { Env } from "../common/env"; import { Env } from "../common/env";
import exec from "../common/exec"; import exec from "../common/exec";
import { System } from "../common/system"; import { System } from "../common/system";
@@ -62,10 +62,10 @@ export class Android extends Sdk {
} }
prepare_ndk(): Promise<any> { prepare_ndk(): Promise<any> {
// ToDo: support windows // TODO: support windows
const url = `${this.repository}/android-ndk-${this.ndk_version}-${System.os}-x86_64.zip`; const url = `${this.repository}/android-ndk-${this.ndk_version}-${System.os}-x86_64.zip`;
this.log.d("download: *%s*", url); this.log.d("download: *%s*", url);
return Downloader.download(url, this.ndk_home, 1, true); return download(url, this.ndk_home, { executable: true });
} }
sdkmanager(packages: string[]) { sdkmanager(packages: string[]) {
+3 -1
View File
@@ -1,6 +1,8 @@
export * from "./common/download";
export * from "./common/log";
export * from "./common/output";
export * from "./common/system"; export * from "./common/system";
export * from "./core/config"; export * from "./core/config";
export * from "./common/download";
export * from "./core/sdk"; export * from "./core/sdk";
export * from "./core/type"; export * from "./core/type";
export * from "./impl/android"; export * from "./impl/android";
+2 -2
View File
@@ -3,7 +3,7 @@ import { existsSync, mkdtempSync } from "node:fs";
import { tmpdir } from "node:os"; import { tmpdir } from "node:os";
import { join } from "node:path"; import { join } from "node:path";
import test from "node:test"; import test from "node:test";
import { Downloader } from "../dist/index.mjs"; import { download } from "../dist/index.mjs";
const resolveTargetDir = () => mkdtempSync(join(tmpdir(), "test-")); const resolveTargetDir = () => mkdtempSync(join(tmpdir(), "test-"));
@@ -16,7 +16,7 @@ const TEST_DATA = {
for (const [url, file] of Object.entries(TEST_DATA)) { for (const [url, file] of Object.entries(TEST_DATA)) {
test(`Test download: ${url}`, async () => { test(`Test download: ${url}`, async () => {
const target = resolveTargetDir(); const target = resolveTargetDir();
await Downloader.download(url, target); await download(url, target);
assert(existsSync(join(target, file))); assert(existsSync(join(target, file)));
}); });
} }