diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..702b4c1 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,7 @@ +[*] +end_of_line = lf +insert_final_newline = true +indent_style = space +indent_size = 2 +charset = utf-8 +trim_trailing_whitespace = true \ No newline at end of file diff --git a/deno.lock b/deno.lock index 70f0fef..abbdbbb 100644 --- a/deno.lock +++ b/deno.lock @@ -40,6 +40,7 @@ "https://deno.land/x/transform@v0.4.0/transformers/zlib/gzip.ts": "8008eb620908025c9768bf2fe43c542b122224deb13ce37034237ed44fa97827", "https://deno.land/x/transform@v0.4.0/transformers/zlib/mod.ts": "ed83911768d491342fb3e09a9b23217c9345ce0b721c69346ed53416d203e195", "https://deno.land/x/transform@v0.4.0/transformers/zlib/pako.js": "f7077bb75f3f8a479fc945cffa93faa113be1d6c1d1c6d8566d35a6406ad40b8", + "https://deno.land/x/yaml@v2.2.1/util.js": "2f1db26f6cc426ef698210b592f40cec49be1b6c4b34e7f2d61904786242bd85", "https://raw.githubusercontent.com/marcosc90/encoding-wasm/v0.1.0/dist/encoding_wasm.js": "d7bb9e589b3505f75d220eadb127d98a3b0ef19a884bafc8f94db7a90a24eacf", "https://raw.githubusercontent.com/marcosc90/encoding-wasm/v0.1.0/dist/encoding_wasm_bytes.js": "c826533d17099db2112c9ddb3cbcd013f7afef5273f4666ac7e0196cc6424659", "https://raw.githubusercontent.com/marcosc90/encoding-wasm/v0.1.0/mod.ts": "0ebef5c2b134fa6b80ebd52242b23cd19b0dd8d661d07d3fb6cef7b971da99f7" diff --git a/deps.ts b/deps.ts index 704d559..1e5752e 100644 --- a/deps.ts +++ b/deps.ts @@ -1,12 +1,9 @@ -export { - type TarMeta, - Untar, -} from "https://deno.land/std@0.144.0/archive/tar.ts"; +export { Untar } from "https://deno.land/std@0.144.0/archive/tar.ts"; export { copy, readerFromStreamReader, } from "https://deno.land/std@0.144.0/streams/mod.ts"; -export { join } from "https://deno.land/std@0.144.0/path/mod.ts"; +export { globToRegExp, join } from "https://deno.land/std@0.144.0/path/mod.ts"; export { default as cacheDir, } from "https://deno.land/x/cache_dir@0.2.0/mod.ts"; diff --git a/mod.d.ts b/mod.d.ts new file mode 100644 index 0000000..a6e7723 --- /dev/null +++ b/mod.d.ts @@ -0,0 +1,34 @@ +export type ReadStrategy = + | "tar" + | "targz" + | "github"; + +export type ReadPredicate = ( + entry: ConfigEntry, +) => AsyncGenerator; + +export type GlobLike = string | RegExp; +export interface ConfigEntry { + strategy?: ReadStrategy; + read?: ReadPredicate; + source: string; + output: string; + pick: GlobLike[]; +} + +export type PickConfig = ConfigEntry[]; + +export interface FileMeta extends Deno.Reader { + fileName?: string; +} + +export interface ReadableEntry extends ConfigEntry { + read: () => AsyncGenerator; +} + +export type PathLike = string | URL; +export interface GithubPickOptions { + repo: string; + version: string; + pick: RegExp[]; +} diff --git a/mod.ts b/mod.ts index 79d7260..ea1a7c4 100644 --- a/mod.ts +++ b/mod.ts @@ -1,29 +1,118 @@ import { cacheDir, copy, + globToRegExp, GzDecoder, join, readerFromStreamReader, - TarMeta, Transform, Untar, } from "./deps.ts"; +import type { + ConfigEntry, + GithubPickOptions, + PathLike, + PickConfig, + ReadableEntry, + ReadPredicate, + ReadStrategy, +} from "./mod.d.ts"; export const CACHE_DIR = join((await cacheDir()) ?? ".cache", "__pick_cache0"); await Deno.mkdir(CACHE_DIR).catch((_) => {}); -type TarEntry = TarMeta & Deno.Reader; -interface GithubPickOptions { - repo: string; - version: string; - pick: RegExp[]; +/** + * Read config and write files + */ +export async function write(entires: PickConfig) { + const _entries: ReadableEntry[] = entires.map(asReadable$); + for (const entry of _entries) { + const output = entry.output; + await Deno.mkdir(output, { recursive: true }); + for await (const reader of entry.read()) { + const file = await Deno.open( + join(output, filename(reader.fileName ?? "")), + { create: true, write: true }, + ); + await copy(reader, file); + file.close(); + } + } +} + +/** + * Decorator which makes config entries readable + */ +function asReadable$(entry: ConfigEntry): ReadableEntry { + // convert globs to regex if required + const pick = entry.pick.map((p) => { + if (typeof p === "string") { + return globToRegExp(p); + } + return p; + }); + + // if read is a function, use it + let read: ReadableEntry["read"]; + if (typeof entry.read === "function") { + read = () => (entry.read as ReadPredicate)(entry); + } else { + // otherwise, use a specific strategy + const strategy = entry.strategy ?? autoStrategy$(entry.source); + switch (strategy) { + case "tar": { + read = () => tarPickFiles(entry.source, pick); + break; + } + case "targz": { + read = () => tarGzPickFiles(entry.source, pick); + break; + } + case "github": { + const [repo, version] = entry.source.split("@"); + if (!version) { + throw new Error( + `Invalid source format: ${entry.source} + Expected: @`, + ); + } + read = () => githubPick({ repo, version, pick }); + break; + } + default: { + throw new Error(`Unknown strategy: ${entry.strategy}`); + } + } + } + return { + ...entry, + read, + }; +} + +function autoStrategy$(source: string): ReadStrategy { + if (source.endsWith(".tar")) { + return "tar"; + } else if (source.endsWith(".tar.gz")) { + return "targz"; + } else if (source.includes("@")) { + return "github"; + } else { + throw new Error(`Unknown strategy for source: ${source}`); + } +} + +function filename(path: string) { + return path.split("/").pop() ?? ""; } /** * Pick files from a github repo. * @param {GithubPickOptions} opts - { repo: "denoland/deno", version: "v1.0.0", pick: [/\.ts$/] } */ -export async function* githubPick({ repo, version, pick }: GithubPickOptions) { +export async function* githubPick( + { repo, version, pick }: GithubPickOptions, +): ReturnType { await Deno.mkdir(join(CACHE_DIR, repo.split("/").shift() ?? "")).catch( (_) => {}, ); @@ -41,11 +130,10 @@ export async function* githubPick({ repo, version, pick }: GithubPickOptions) { * Reads *.tar.gz file from a version tag and returns generator of the files that match the pick regex. * * @param {GithubPickOptions} opts - { repo: "denoland/deno", version: "v1.0.0", pick: [/\.ts$/] } - * @returns {AsyncGenerator} */ export async function* githubPickFiles( { repo, version, pick }: GithubPickOptions, -): AsyncGenerator { +): ReturnType { const targz = await fetch( `https://github.com/${repo}/archive/refs/tags/${version}.tar.gz`, ); @@ -59,13 +147,16 @@ export async function* githubPickFiles( /** * Pick files from a tar.gz archive. * - * @param {Deno.Reader} tar - Deno.open('archive.tar.gz') + * @param {Deno.Reader} targz - Deno.open('archive.tar.gz') * @param {RegExp[]} pick - [ /\.ts$/ ] */ export async function* tarGzPickFiles( - targz: Deno.Reader | Deno.FsFile, + targz: Deno.Reader | Deno.FsFile | PathLike, pick: RegExp[], -) { +): ReturnType { + if (typeof targz === "string" || targz instanceof URL) { + targz = await Deno.open(targz); + } const untar = new Untar( Transform.newReader( targz, @@ -79,15 +170,35 @@ export async function* tarGzPickFiles( } } +/** + * Pick files from a tar.gz archive. + * + * @param {Deno.Reader} tar - Deno.open('archive.tar') + * @param {RegExp[]} pick - [ /\.ts$/ ] + */ +export async function* tarPickFiles( + tar: Deno.Reader | Deno.FsFile | PathLike, + pick: RegExp[], +): ReturnType { + if (typeof tar === "string" || tar instanceof URL) { + tar = await Deno.open(tar); + } + const untar = new Untar(tar); + for await (const entry of untar) { + if (entry.type === "file" && pick.some((re) => re.test(entry.fileName))) { + yield entry; + } + } +} + /** * Writes fetch body to a cache file. - * @param body - * @param name - * @param opts */ export async function putFetchCache(body: Response["body"], name: string) { const reader = readerFromStreamReader(body!.getReader()); - const writer = await Deno.open(join(CACHE_DIR, name), { + const _file = join(CACHE_DIR, name); + await Deno.remove(_file).catch((_) => {}); + const writer = await Deno.open(_file, { write: true, create: true, }); @@ -97,8 +208,6 @@ export async function putFetchCache(body: Response["body"], name: string) { /** * Get either the path to the cache file or null if it doesn't exist. - * @param {string} name - * @returns {Promise} */ export async function getFetchCache(name: string) { try { @@ -112,22 +221,7 @@ export async function getFetchCache(name: string) { /** * Clean the fetch cache. - * @returns {Promise} */ export async function cleanFetchCache() { - return Deno.remove(CACHE_DIR, { recursive: true }); -} - -export async function writeTarEntry( - entry: TarEntry, - dir: string, - formatWritePath: (path: string) => string = (path) => path, -) { - const path = formatWritePath(entry.fileName); - const file = await Deno.open(dir + path, { - create: true, - write: true, - }); - await copy(entry, file); - file.close(); + return await Deno.remove(CACHE_DIR, { recursive: true }); } diff --git a/pick.ts b/pick.ts deleted file mode 100644 index e69de29..0000000 diff --git a/pickit.ts b/pickit.ts new file mode 100644 index 0000000..0f2d2d2 --- /dev/null +++ b/pickit.ts @@ -0,0 +1,81 @@ +// command line interface for pickit +// semantics: +// - pickit [source] [outputDir] [glob1] [glob2]... +// - pickit ./config.ts (default: ./.pickit.ts) + +import { cleanFetchCache, write } from "./mod.ts"; + +if (Deno.args.length == 1) { + if (Deno.args[0] == "clean") { + await cleanFetchCache(); + } else { + const config = await import(Deno.args[0]); + await write(config.default); + } +} else if (Deno.args.length >= 3) { + const [source, output, ...pick] = Deno.args; + await write([{ source, output, pick }]); +} else { + console.log( + ` + %cPickIt + `, + "font-size: 1.5em; font-weight: bold", + ); + console.log(` + This utility helps you to extract files from tarballs and github repos using glob syntax or regular expressions. + You can use either a config file or command line arguments. + `); + console.log( + ` + %cUsage: + `, + "font-size: 1.5em; font-weight: bold", + ); + console.log( + ` + pickit %c[source] [outputDir] [glob1] [glob2]... + %cpickit %c./config.ts + `, + "color: #ccb;", + "", + "color: #ccb;", + ); + console.log( + ` + %cExample: + `, + "font-size: 1.5em; font-weight: bold", + ); + console.log( + ` + %cpickit nesterow/pickit@v0.0.1 scripts *.d.ts **/tests/*.ts + `, + "color: #aab;", + ); + console.log( + ` + %cConfiguring (ts): + `, + "font-size: 1.5em; font-weight: bold", + ); + console.log( + `%c + import type { PickConfig } from "https://deno.land/x/pickit/mod.d.ts"; + export default [ + { + source: "username/repo@version", + output: "./outputDir", + pick: [ + /^.*\/base\/.*\.css$/, + "/src/index.js", + "/src/**/*.yaml" + ], + }, + ] as PickConfig; + + + `, + "color: #aab;", + ); +} diff --git a/tests/.pick.yaml b/tests/.pick.yaml deleted file mode 100644 index e69de29..0000000 diff --git a/tests/config_mock.ts b/tests/config_mock.ts new file mode 100644 index 0000000..b23aa04 --- /dev/null +++ b/tests/config_mock.ts @@ -0,0 +1,11 @@ +import type { PickConfig } from "../mod.d.ts"; + +export default [ + { + source: "saadeghi/daisyui@v2.47.0", + output: ".basestyles", + pick: [ + /^.*\/base\/.*\.css$/, + ], + }, +] as PickConfig; diff --git a/tests/mod_test.ts b/tests/mod_test.ts index a3cf667..5088f56 100644 --- a/tests/mod_test.ts +++ b/tests/mod_test.ts @@ -1,28 +1,93 @@ import { assert } from "https://deno.land/std@0.144.0/_util/assert.ts"; -import {join} from '../deps.ts'; +import { join } from "../deps.ts"; import * as mod from "../mod.ts"; +import type { FileMeta, PickConfig } from "../mod.d.ts"; const { test } = Deno; test("mod", async (t) => { const repo = "saadeghi/daisyui"; const version = "v2.47.0"; - await t.step("githubPick", async () => { + + await t.step("mod.githubPick", async () => { const pick = [ /^.*\/base\/.*\.css$/, /src\/index\.js/, ]; - const files = []; + const files: FileMeta[] = []; for await (const file of mod.githubPick({ repo, version, pick })) { files.push(file); } assert(files.length > 0); - assert(files.filter(f => f.fileName.endsWith(".js")).length == 1); - assert(files.filter(f => f.fileName.endsWith(".css")).length == 2); + assert(files.filter((f) => f.fileName?.endsWith(".js")).length == 1); + assert(files.filter((f) => f.fileName?.endsWith(".css")).length == 2); }); - await t.step("is cached", async () => { + + await t.step("mod.getFetchCache", async () => { const name = `${repo}@${version}`; const cached = await mod.getFetchCache(name); assert(cached?.endsWith(version)); assert(cached == join(mod.CACHE_DIR, name)); }); + + await t.step("mod.write: autoStrategy", async () => { + const output = ".daisyui"; + const config: PickConfig = [ + { + source: `${repo}@${version}`, + output, + pick: [ + /^.*\/base\/.*\.css$/, + /src\/index\.js/, + ], + }, + ]; + await mod.write(config); + const files = await Deno.readDir(output); + for await (const file of files) { + assert(file.isFile); + assert(file.name.endsWith(".js") || file.name.endsWith(".css")); + await Deno.remove(join(output, file.name)); + } + await Deno.remove(output); + }); + + await t.step("mod.write: use glob", async () => { + const output = ".daisyui"; + const config: PickConfig = [ + { + source: `${repo}@${version}`, + output, + pick: [ + "**/base/*.css", + "src/index.js", + ], + }, + ]; + await mod.write(config); + const files = await Deno.readDir(output); + for await (const file of files) { + assert(file.isFile); + assert(file.name.endsWith(".js") || file.name.endsWith(".css")); + await Deno.remove(join(output, file.name)); + } + await Deno.remove(output); + }); +}); + +test("pickit: cmd line interface", async () => { + const p = await Deno.run({ + cmd: [ + "deno", + "run", + "-A", + "pickit.ts", + "saadeghi/daisyui@v2.47.0", + "daisyui", + "**/src/index.js", + "**/base/*.css", + ], + }); + const status = await p.status(); + assert(status.success); + await Deno.remove("daisyui", { recursive: true }); });