x/pickit
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

add tests and command line interface

Browse source
This commit is contained in:
Anton Nesterov 2023-02-11 14:13:48 +03:00
parent a4f67c1495
commit f7b17d56f3
10 changed files with 335 additions and 45 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
.editorconfig Normal file
View file

@ -0,0 +1,7 @@
[*]
end_of_line = lf
insert_final_newline = true
indent_style = space
indent_size = 2
charset = utf-8
trim_trailing_whitespace = true

1
deno.lock
View file

@ -40,6 +40,7 @@
"https://deno.land/x/transform@v0.4.0/transformers/zlib/gzip.ts": "8008eb620908025c9768bf2fe43c542b122224deb13ce37034237ed44fa97827",
"https://deno.land/x/transform@v0.4.0/transformers/zlib/mod.ts": "ed83911768d491342fb3e09a9b23217c9345ce0b721c69346ed53416d203e195",
"https://deno.land/x/transform@v0.4.0/transformers/zlib/pako.js": "f7077bb75f3f8a479fc945cffa93faa113be1d6c1d1c6d8566d35a6406ad40b8",
"https://deno.land/x/yaml@v2.2.1/util.js": "2f1db26f6cc426ef698210b592f40cec49be1b6c4b34e7f2d61904786242bd85",
"https://raw.githubusercontent.com/marcosc90/encoding-wasm/v0.1.0/dist/encoding_wasm.js": "d7bb9e589b3505f75d220eadb127d98a3b0ef19a884bafc8f94db7a90a24eacf",
"https://raw.githubusercontent.com/marcosc90/encoding-wasm/v0.1.0/dist/encoding_wasm_bytes.js": "c826533d17099db2112c9ddb3cbcd013f7afef5273f4666ac7e0196cc6424659",
"https://raw.githubusercontent.com/marcosc90/encoding-wasm/v0.1.0/mod.ts": "0ebef5c2b134fa6b80ebd52242b23cd19b0dd8d661d07d3fb6cef7b971da99f7"

7
deps.ts
View file

@ -1,12 +1,9 @@
export {
type TarMeta,
Untar,
} from "https://deno.land/std@0.144.0/archive/tar.ts";
export { Untar } from "https://deno.land/std@0.144.0/archive/tar.ts";
export {
copy,
readerFromStreamReader,
} from "https://deno.land/std@0.144.0/streams/mod.ts";
export { join } from "https://deno.land/std@0.144.0/path/mod.ts";
export { globToRegExp, join } from "https://deno.land/std@0.144.0/path/mod.ts";
export {
default as cacheDir,
} from "https://deno.land/x/cache_dir@0.2.0/mod.ts";

34
mod.d.ts vendored Normal file
View file

@ -0,0 +1,34 @@
export type ReadStrategy =
| "tar"
| "targz"
| "github";
export type ReadPredicate = (
entry: ConfigEntry,
) => AsyncGenerator<Deno.Reader | Deno.Reader & FileMeta>;
export type GlobLike = string | RegExp;
export interface ConfigEntry {
strategy?: ReadStrategy;
read?: ReadPredicate;
source: string;
output: string;
pick: GlobLike[];
}
export type PickConfig = ConfigEntry[];
export interface FileMeta extends Deno.Reader {
fileName?: string;
}
export interface ReadableEntry extends ConfigEntry {
read: () => AsyncGenerator<FileMeta>;
}
export type PathLike = string | URL;
export interface GithubPickOptions {
repo: string;
version: string;
pick: RegExp[];
}

162
mod.ts
View file

@ -1,29 +1,118 @@
import {
cacheDir,
copy,
globToRegExp,
GzDecoder,
join,
readerFromStreamReader,
TarMeta,
Transform,
Untar,
} from "./deps.ts";
import type {
ConfigEntry,
GithubPickOptions,
PathLike,
PickConfig,
ReadableEntry,
ReadPredicate,
ReadStrategy,
} from "./mod.d.ts";
export const CACHE_DIR = join((await cacheDir()) ?? ".cache", "__pick_cache0");
await Deno.mkdir(CACHE_DIR).catch((_) => {});
type TarEntry = TarMeta & Deno.Reader;
interface GithubPickOptions {
repo: string;
version: string;
pick: RegExp[];
/**
* Read config and write files
*/
export async function write(entires: PickConfig) {
const _entries: ReadableEntry[] = entires.map(asReadable$);
for (const entry of _entries) {
const output = entry.output;
await Deno.mkdir(output, { recursive: true });
for await (const reader of entry.read()) {
const file = await Deno.open(
join(output, filename(reader.fileName ?? "")),
{ create: true, write: true },
);
await copy(reader, file);
file.close();
}
}
}
/**
* Decorator which makes config entries readable
*/
function asReadable$(entry: ConfigEntry): ReadableEntry {
// convert globs to regex if required
const pick = entry.pick.map((p) => {
if (typeof p === "string") {
return globToRegExp(p);
}
return p;
});
// if read is a function, use it
let read: ReadableEntry["read"];
if (typeof entry.read === "function") {
read = () => (entry.read as ReadPredicate)(entry);
} else {
// otherwise, use a specific strategy
const strategy = entry.strategy ?? autoStrategy$(entry.source);
switch (strategy) {
case "tar": {
read = () => tarPickFiles(entry.source, pick);
break;
}
case "targz": {
read = () => tarGzPickFiles(entry.source, pick);
break;
}
case "github": {
const [repo, version] = entry.source.split("@");
if (!version) {
throw new Error(
`Invalid source format: ${entry.source}
Expected: <repo>@<version>`,
);
}
read = () => githubPick({ repo, version, pick });
break;
}
default: {
throw new Error(`Unknown strategy: ${entry.strategy}`);
}
}
}
return {
...entry,
read,
};
}
function autoStrategy$(source: string): ReadStrategy {
if (source.endsWith(".tar")) {
return "tar";
} else if (source.endsWith(".tar.gz")) {
return "targz";
} else if (source.includes("@")) {
return "github";
} else {
throw new Error(`Unknown strategy for source: ${source}`);
}
}
function filename(path: string) {
return path.split("/").pop() ?? "";
}
/**
* Pick files from a github repo.
* @param {GithubPickOptions} opts - { repo: "denoland/deno", version: "v1.0.0", pick: [/\.ts$/] }
*/
export async function* githubPick({ repo, version, pick }: GithubPickOptions) {
export async function* githubPick(
{ repo, version, pick }: GithubPickOptions,
): ReturnType<ReadableEntry["read"]> {
await Deno.mkdir(join(CACHE_DIR, repo.split("/").shift() ?? "")).catch(
(_) => {},
);
@ -41,11 +130,10 @@ export async function* githubPick({ repo, version, pick }: GithubPickOptions) {
* Reads *.tar.gz file from a version tag and returns generator of the files that match the pick regex.
*
* @param {GithubPickOptions} opts - { repo: "denoland/deno", version: "v1.0.0", pick: [/\.ts$/] }
* @returns {AsyncGenerator<TarEntry>}
*/
export async function* githubPickFiles(
{ repo, version, pick }: GithubPickOptions,
): AsyncGenerator<TarEntry> {
): ReturnType<ReadableEntry["read"]> {
const targz = await fetch(
`https://github.com/${repo}/archive/refs/tags/${version}.tar.gz`,
);
@ -59,13 +147,16 @@ export async function* githubPickFiles(
/**
* Pick files from a tar.gz archive.
*
* @param {Deno.Reader} tar - Deno.open('archive.tar.gz')
* @param {Deno.Reader} targz - Deno.open('archive.tar.gz')
* @param {RegExp[]} pick - [ /\.ts$/ ]
*/
export async function* tarGzPickFiles(
targz: Deno.Reader | Deno.FsFile,
targz: Deno.Reader | Deno.FsFile | PathLike,
pick: RegExp[],
) {
): ReturnType<ReadableEntry["read"]> {
if (typeof targz === "string" || targz instanceof URL) {
targz = await Deno.open(targz);
}
const untar = new Untar(
Transform.newReader(
targz,
@ -79,15 +170,35 @@ export async function* tarGzPickFiles(
}
}
/**
* Pick files from a tar.gz archive.
*
* @param {Deno.Reader} tar - Deno.open('archive.tar')
* @param {RegExp[]} pick - [ /\.ts$/ ]
*/
export async function* tarPickFiles(
tar: Deno.Reader | Deno.FsFile | PathLike,
pick: RegExp[],
): ReturnType<ReadableEntry["read"]> {
if (typeof tar === "string" || tar instanceof URL) {
tar = await Deno.open(tar);
}
const untar = new Untar(tar);
for await (const entry of untar) {
if (entry.type === "file" && pick.some((re) => re.test(entry.fileName))) {
yield entry;
}
}
}
/**
* Writes fetch body to a cache file.
* @param body
* @param name
* @param opts
*/
export async function putFetchCache(body: Response["body"], name: string) {
const reader = readerFromStreamReader(body!.getReader());
const writer = await Deno.open(join(CACHE_DIR, name), {
const _file = join(CACHE_DIR, name);
await Deno.remove(_file).catch((_) => {});
const writer = await Deno.open(_file, {
write: true,
create: true,
});
@ -97,8 +208,6 @@ export async function putFetchCache(body: Response["body"], name: string) {
/**
* Get either the path to the cache file or null if it doesn't exist.
* @param {string} name
* @returns {Promise<string | null>}
*/
export async function getFetchCache(name: string) {
try {
@ -112,22 +221,7 @@ export async function getFetchCache(name: string) {
/**
* Clean the fetch cache.
* @returns {Promise<void>}
*/
export async function cleanFetchCache() {
return Deno.remove(CACHE_DIR, { recursive: true });
}
export async function writeTarEntry(
entry: TarEntry,
dir: string,
formatWritePath: (path: string) => string = (path) => path,
) {
const path = formatWritePath(entry.fileName);
const file = await Deno.open(dir + path, {
create: true,
write: true,
});
await copy(entry, file);
file.close();
return await Deno.remove(CACHE_DIR, { recursive: true });
}

0
pick.ts
View file

81
pickit.ts Normal file
View file

@ -0,0 +1,81 @@
// command line interface for pickit
// semantics:
// - pickit [source] [outputDir] [glob1] [glob2]...
// - pickit ./config.ts (default: ./.pickit.ts)
import { cleanFetchCache, write } from "./mod.ts";
if (Deno.args.length == 1) {
if (Deno.args[0] == "clean") {
await cleanFetchCache();
} else {
const config = await import(Deno.args[0]);
await write(config.default);
}
} else if (Deno.args.length >= 3) {
const [source, output, ...pick] = Deno.args;
await write([{ source, output, pick }]);
} else {
console.log(
`
%cPickIt
`,
"font-size: 1.5em; font-weight: bold",
);
console.log(`
This utility helps you to extract files from tarballs and github repos using glob syntax or regular expressions.
You can use either a config file or command line arguments.
`);
console.log(
`
%cUsage:
`,
"font-size: 1.5em; font-weight: bold",
);
console.log(
`
pickit %c[source] [outputDir] [glob1] [glob2]...
%cpickit %c./config.ts
`,
"color: #ccb;",
"",
"color: #ccb;",
);
console.log(
`
%cExample:
`,
"font-size: 1.5em; font-weight: bold",
);
console.log(
`
%cpickit nesterow/pickit@v0.0.1 scripts *.d.ts **/tests/*.ts
`,
"color: #aab;",
);
console.log(
`
%cConfiguring (ts):
`,
"font-size: 1.5em; font-weight: bold",
);
console.log(
`%c
import type { PickConfig } from "https://deno.land/x/pickit/mod.d.ts";
export default [
{
source: "username/repo@version",
output: "./outputDir",
pick: [
/^.*\/base\/.*\.css$/,
"/src/index.js",
"/src/**/*.yaml"
],
},
] as PickConfig;
`,
"color: #aab;",
);
}

0
tests/.pick.yaml
View file

11
tests/config_mock.ts Normal file
View file

@ -0,0 +1,11 @@
import type { PickConfig } from "../mod.d.ts";
export default [
{
source: "saadeghi/daisyui@v2.47.0",
output: ".basestyles",
pick: [
/^.*\/base\/.*\.css$/,
],
},
] as PickConfig;

77
tests/mod_test.ts
View file

@ -1,28 +1,93 @@
import { assert } from "https://deno.land/std@0.144.0/_util/assert.ts";
import {join} from '../deps.ts';
import { join } from "../deps.ts";
import * as mod from "../mod.ts";
import type { FileMeta, PickConfig } from "../mod.d.ts";
const { test } = Deno;
test("mod", async (t) => {
const repo = "saadeghi/daisyui";
const version = "v2.47.0";
await t.step("githubPick", async () => {
await t.step("mod.githubPick", async () => {
const pick = [
/^.*\/base\/.*\.css$/,
/src\/index\.js/,
];
const files = [];
const files: FileMeta[] = [];
for await (const file of mod.githubPick({ repo, version, pick })) {
files.push(file);
}
assert(files.length > 0);
assert(files.filter(f => f.fileName.endsWith(".js")).length == 1);
assert(files.filter(f => f.fileName.endsWith(".css")).length == 2);
assert(files.filter((f) => f.fileName?.endsWith(".js")).length == 1);
assert(files.filter((f) => f.fileName?.endsWith(".css")).length == 2);
});
await t.step("is cached", async () => {
await t.step("mod.getFetchCache", async () => {
const name = `${repo}@${version}`;
const cached = await mod.getFetchCache(name);
assert(cached?.endsWith(version));
assert(cached == join(mod.CACHE_DIR, name));
});
await t.step("mod.write: autoStrategy", async () => {
const output = ".daisyui";
const config: PickConfig = [
{
source: `${repo}@${version}`,
output,
pick: [
/^.*\/base\/.*\.css$/,
/src\/index\.js/,
],
},
];
await mod.write(config);
const files = await Deno.readDir(output);
for await (const file of files) {
assert(file.isFile);
assert(file.name.endsWith(".js") || file.name.endsWith(".css"));
await Deno.remove(join(output, file.name));
}
await Deno.remove(output);
});
await t.step("mod.write: use glob", async () => {
const output = ".daisyui";
const config: PickConfig = [
{
source: `${repo}@${version}`,
output,
pick: [
"**/base/*.css",
"src/index.js",
],
},
];
await mod.write(config);
const files = await Deno.readDir(output);
for await (const file of files) {
assert(file.isFile);
assert(file.name.endsWith(".js") || file.name.endsWith(".css"));
await Deno.remove(join(output, file.name));
}
await Deno.remove(output);
});
});
test("pickit: cmd line interface", async () => {
const p = await Deno.run({
cmd: [
"deno",
"run",
"-A",
"pickit.ts",
"saadeghi/daisyui@v2.47.0",
"daisyui",
"**/src/index.js",
"**/base/*.css",
],
});
const status = await p.status();
assert(status.success);
await Deno.remove("daisyui", { recursive: true });
});