commit 61d5c0edeb39f65e498fdfe14ff96b124e6df8e8 Author: Andrew Boldi Date: Sat Apr 25 23:37:07 2026 -0700 Phase 1: tarball-based seed via streaming fetch Co-Authored-By: Claude Opus 4.7 (1M context) diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..3951cb8 --- /dev/null +++ b/.gitignore @@ -0,0 +1,12 @@ +node_modules/ +.DS_Store +*.log +.env +data.json + +# IDE +.vscode/ +.idea/ + +# Build artifacts (main.js IS committed for BRAT) +*.map diff --git a/esbuild.config.mjs b/esbuild.config.mjs new file mode 100644 index 0000000..71fa400 --- /dev/null +++ b/esbuild.config.mjs @@ -0,0 +1,32 @@ +import esbuild from "esbuild"; +import process from "process"; + +const banner = `/* +THIS IS A GENERATED/BUNDLED FILE BY ESBUILD +if you want to view the source visit https://github.com/andrewboldi/obsidian-vault-sync +*/ +`; + +const prod = process.argv[2] === "production"; + +const context = await esbuild.context({ + banner: { js: banner }, + entryPoints: ["src/main.ts"], + bundle: true, + external: ["obsidian", "electron"], + format: "cjs", + target: "es2020", + logLevel: "info", + sourcemap: prod ? false : "inline", + treeShaking: true, + platform: "browser", + minify: prod, + outfile: "main.js", +}); + +if (prod) { + await context.rebuild(); + process.exit(0); +} else { + await context.watch(); +} diff --git a/main.js b/main.js new file mode 100644 index 0000000..f6c1b7b --- /dev/null +++ b/main.js @@ -0,0 +1,6 @@ +/* +THIS IS A GENERATED/BUNDLED FILE BY ESBUILD +if you want to view the source visit https://github.com/andrewboldi/obsidian-vault-sync +*/ + +"use strict";var $=Object.defineProperty;var O=Object.getOwnPropertyDescriptor;var G=Object.getOwnPropertyNames;var H=Object.prototype.hasOwnProperty;var K=(n,t)=>{for(var e in t)$(n,e,{get:t[e],enumerable:!0})},M=(n,t,e,i)=>{if(t&&typeof t=="object"||typeof t=="function")for(let r of G(t))!H.call(n,r)&&r!==e&&$(n,r,{get:()=>t[r],enumerable:!(i=O(t,r))||i.enumerable});return n};var N=n=>M($({},"__esModule",{value:!0}),n);var j={};K(j,{default:()=>b});module.exports=N(j);var w=require("obsidian");var p=require("obsidian"),T={pat:"",repoUrl:"",branch:"main",lastCommitSha:""},y=class extends p.PluginSettingTab{constructor(t,e){super(t,e),this.plugin=e}display(){let{containerEl:t}=this;t.empty(),t.createEl("h2",{text:"Vault Sync (REST)"}),new p.Setting(t).setName("GitHub Personal Access Token").setDesc("Fine-grained or classic PAT with repo (contents: read/write) scope on the target repo.").addText(e=>{e.inputEl.type="password",e.setPlaceholder("ghp_\u2026").setValue(this.plugin.settings.pat).onChange(async i=>{this.plugin.settings.pat=i.trim(),await this.plugin.saveSettings()})}),new p.Setting(t).setName("Repository URL").setDesc("e.g. https://github.com/owner/repo").addText(e=>e.setPlaceholder("https://github.com/owner/repo").setValue(this.plugin.settings.repoUrl).onChange(async i=>{this.plugin.settings.repoUrl=i.trim(),await this.plugin.saveSettings()})),new p.Setting(t).setName("Branch").setDesc("Branch to sync (default: main)").addText(e=>e.setPlaceholder("main").setValue(this.plugin.settings.branch).onChange(async i=>{this.plugin.settings.branch=i.trim()||"main",await this.plugin.saveSettings()})),new p.Setting(t).setName("Last commit SHA").setDesc("Set after a successful seed/pull. Read-only \u2014 used as the local sync base.").addText(e=>{e.setValue(this.plugin.settings.lastCommitSha||"(none)").setDisabled(!0)})}};var k=require("obsidian");var F="obsidian-vault-sync",C="https://api.github.com";function x(n){return{Authorization:`token ${n}`,"User-Agent":F,Accept:"application/vnd.github+json","X-GitHub-Api-Version":"2022-11-28"}}function V(n){if(!n)throw new Error("Repository URL is empty");let t=n.trim().replace(/\.git$/,"").replace(/\/+$/,""),e=t.match(/^https?:\/\/github\.com\/([^/]+)\/([^/]+)$/i);if(e||(e=t.match(/^([^/\s]+)\/([^/\s]+)$/)),!e)throw new Error(`Cannot parse repo URL: ${n}`);return{owner:e[1],repo:e[2]}}async function R(n,t,e){let i=`${C}/repos/${n.owner}/${n.repo}/branches/${encodeURIComponent(t)}`,r=await fetch(i,{headers:x(e)});if(!r.ok){let l=await r.text();throw new Error(`GitHub branch lookup failed: ${r.status} ${r.statusText} \u2014 ${l.slice(0,300)}`)}let s=(await r.json())?.commit?.sha;if(!s)throw new Error("GitHub response missing commit.sha");return s}function B(n,t){return`${C}/repos/${n.owner}/${n.repo}/tarball/${t}`}function U(n,t,e){let i=t,r=t+e;for(;i0)break;continue}i+=String.fromCharCode(o)}return i?parseInt(i,8)|0:0}function z(n,t,e){for(let i=t;ithis.buf.length-this.start)if(this.start>0&&e<=this.buf.length)this.buf.copyWithin(0,this.start,this.end),this.end-=this.start,this.start=0;else{let i=Math.max(this.buf.length*2,65536);for(;ithis.buf.length/2&&(this.buf.copyWithin(0,this.start,this.end),this.end-=this.start,this.start=0),e}skip(t){this.start+=t,this.start>this.buf.length/2&&(this.buf.copyWithin(0,this.start,this.end),this.end-=this.start,this.start=0)}};async function*E(n){let t=n.getReader(),e=new A,i=0,r=!1;async function o(s){for(;e.length=s;l&&l.length&&e.push(l)}return!0}for(;await o(512);){let s=e.peek(512);if(z(s,0,512)){if(e.skip(512),i++,i>=2)break;continue}i=0;let l=U(s,0,100),c=W(s,124,12),m=s[156],h=m===0?"0":String.fromCharCode(m),d=U(s,257,6),g="";(d==="ustar"||d.startsWith("ustar"))&&(g=U(s,345,155));let a=g?`${g}/${l}`:l;e.skip(512);let u=c>0?Math.ceil(c/512)*512:0;if(h!=="0"&&h!=="5"&&h!=="\0"){let f=u;for(;f>0;){if(!await o(Math.min(f,512)))return;let v=Math.min(f,e.length);e.skip(v),f-=v}continue}if(h==="5"){yield{name:a,type:"dir",content:new Uint8Array(0)};continue}if(!await o(u)){let f=Math.min(e.length,c);yield{name:a,type:"file",content:e.take(f)};return}let S=e.take(c);u>c&&e.skip(u-c),yield{name:a,type:"file",content:S}}}function P(n){return n<1024?`${n} B`:n<1024*1024?`${(n/1024).toFixed(1)} KB`:n<1024*1024*1024?`${(n/(1024*1024)).toFixed(1)} MB`:`${(n/(1024*1024*1024)).toFixed(2)} GB`}function _(n){let t=n.replace(/\\/g,"/"),e=t.indexOf("/");if(e<0)return null;let i=t.slice(e+1);return!i||i==="/"||i.startsWith("/")||i.includes("../")?null:i}async function L(n,t){if(!(!t||t==="/"||t==="."))try{await n.app.vault.adapter.mkdir(t)}catch{}}async function I(n,t){let e=t.split("/");if(e.pop(),e.length===0)return;let i="";for(let r of e)r&&(i=i?`${i}/${r}`:r,await L(n,i))}async function D(n){let{pat:t,repoUrl:e,branch:i}=n.settings;if(!t)throw new Error("Set a GitHub PAT in Vault Sync settings.");if(!e)throw new Error("Set a repo URL in Vault Sync settings.");let r=V(e),o=new k.Notice("Vault Sync: resolving branch\u2026",0),s;try{s=await R(r,i||"main",t)}catch(a){throw o.hide(),a}o.setMessage(`Vault Sync: downloading ${r.owner}/${r.repo}@${s.slice(0,7)}\u2026`);let l=B(r,s),c=await fetch(l,{headers:x(t)});if(!c.ok||!c.body){o.hide();let a=c.body?await c.text().catch(()=>""):"";throw new Error(`Tarball fetch failed: ${c.status} ${c.statusText} \u2014 ${a.slice(0,300)}`)}let m;try{m=c.body.pipeThrough(new DecompressionStream("gzip"))}catch(a){throw o.hide(),new Error("DecompressionStream('gzip') is not available in this WebView. Vault Sync requires iOS 16.4+/modern Chromium. "+String(a))}let h=0,d=0,g=0;for await(let a of E(m)){let u=_(a.name);if(!u)continue;if(a.type==="dir"){await L(n,u.replace(/\/$/,""));continue}await I(n,u);let S=a.content.buffer.slice(a.content.byteOffset,a.content.byteOffset+a.content.byteLength);await n.app.vault.adapter.writeBinary(u,S),h++,d+=a.content.byteLength,h-g>=25&&(o.setMessage(`Vault Sync: extracted ${h} files, ${P(d)}`),g=h)}n.settings.lastCommitSha=s,await n.saveSettings(),o.setMessage(`Vault Sync: seed complete \u2014 ${h} files, ${P(d)} @ ${s.slice(0,7)}`),setTimeout(()=>o.hide(),6e3)}var b=class extends w.Plugin{async onload(){await this.loadSettings(),this.addSettingTab(new y(this.app,this)),this.addCommand({id:"seed-from-github",name:"Seed from GitHub",callback:async()=>{try{await D(this)}catch(t){let e=t instanceof Error?t.message:String(t);console.error("[Vault Sync] seed failed",t),new w.Notice(`Vault Sync seed failed: ${e}`,1e4)}}})}async loadSettings(){let t=await this.loadData()??{};this.settings=Object.assign({},T,t)}async saveSettings(){await this.saveData(this.settings)}}; diff --git a/manifest.json b/manifest.json new file mode 100644 index 0000000..1c2c1ce --- /dev/null +++ b/manifest.json @@ -0,0 +1,10 @@ +{ + "id": "obsidian-vault-sync", + "name": "Vault Sync (REST)", + "version": "0.1.0", + "minAppVersion": "1.4.0", + "description": "Sync Obsidian vault with GitHub via REST API. Works on iOS for any vault size including images.", + "author": "Andrew Boldi", + "authorUrl": "https://github.com/andrewboldi/obsidian-vault-sync", + "isDesktopOnly": false +} diff --git a/package-lock.json b/package-lock.json new file mode 100644 index 0000000..203152e --- /dev/null +++ b/package-lock.json @@ -0,0 +1,633 @@ +{ + "name": "obsidian-vault-sync", + "version": "0.1.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "obsidian-vault-sync", + "version": "0.1.0", + "license": "MIT", + "devDependencies": { + "@types/node": "^20.11.0", + "esbuild": "^0.24.2", + "obsidian": "^1.4.0", + "tslib": "^2.6.2", + "typescript": "^5.3.3" + } + }, + "node_modules/@codemirror/state": { + "version": "6.5.0", + "resolved": "https://registry.npmjs.org/@codemirror/state/-/state-6.5.0.tgz", + "integrity": "sha512-MwBHVK60IiIHDcoMet78lxt6iw5gJOGSbNbOIVBHWVXIH4/Nq1+GQgLLGgI1KlnN86WDXsPudVaqYHKBIx7Eyw==", + "dev": true, + "license": "MIT", + "peer": true, + "dependencies": { + "@marijn/find-cluster-break": "^1.0.0" + } + }, + "node_modules/@codemirror/view": { + "version": "6.38.6", + "resolved": "https://registry.npmjs.org/@codemirror/view/-/view-6.38.6.tgz", + "integrity": "sha512-qiS0z1bKs5WOvHIAC0Cybmv4AJSkAXgX5aD6Mqd2epSLlVJsQl8NG23jCVouIgkh4All/mrbdsf2UOLFnJw0tw==", + "dev": true, + "license": "MIT", + "peer": true, + "dependencies": { + "@codemirror/state": "^6.5.0", + "crelt": "^1.0.6", + "style-mod": "^4.1.0", + "w3c-keyname": "^2.2.4" + } + }, + "node_modules/@esbuild/aix-ppc64": { + "version": "0.24.2", + "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.24.2.tgz", + "integrity": "sha512-thpVCb/rhxE/BnMLQ7GReQLLN8q9qbHmI55F4489/ByVg2aQaQ6kbcLb6FHkocZzQhxc4gx0sCk0tJkKBFzDhA==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "aix" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-arm": { + "version": "0.24.2", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.24.2.tgz", + "integrity": "sha512-tmwl4hJkCfNHwFB3nBa8z1Uy3ypZpxqxfTQOcHX+xRByyYgunVbZ9MzUUfb0RxaHIMnbHagwAxuTL+tnNM+1/Q==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-arm64": { + "version": "0.24.2", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.24.2.tgz", + "integrity": "sha512-cNLgeqCqV8WxfcTIOeL4OAtSmL8JjcN6m09XIgro1Wi7cF4t/THaWEa7eL5CMoMBdjoHOTh/vwTO/o2TRXIyzg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-x64": { + "version": "0.24.2", + "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.24.2.tgz", + "integrity": "sha512-B6Q0YQDqMx9D7rvIcsXfmJfvUYLoP722bgfBlO5cGvNVb5V/+Y7nhBE3mHV9OpxBf4eAS2S68KZztiPaWq4XYw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/darwin-arm64": { + "version": "0.24.2", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.24.2.tgz", + "integrity": "sha512-kj3AnYWc+CekmZnS5IPu9D+HWtUI49hbnyqk0FLEJDbzCIQt7hg7ucF1SQAilhtYpIujfaHr6O0UHlzzSPdOeA==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/darwin-x64": { + "version": "0.24.2", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.24.2.tgz", + "integrity": "sha512-WeSrmwwHaPkNR5H3yYfowhZcbriGqooyu3zI/3GGpF8AyUdsrrP0X6KumITGA9WOyiJavnGZUwPGvxvwfWPHIA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/freebsd-arm64": { + "version": "0.24.2", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.24.2.tgz", + "integrity": "sha512-UN8HXjtJ0k/Mj6a9+5u6+2eZ2ERD7Edt1Q9IZiB5UZAIdPnVKDoG7mdTVGhHJIeEml60JteamR3qhsr1r8gXvg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/freebsd-x64": { + "version": "0.24.2", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.24.2.tgz", + "integrity": "sha512-TvW7wE/89PYW+IevEJXZ5sF6gJRDY/14hyIGFXdIucxCsbRmLUcjseQu1SyTko+2idmCw94TgyaEZi9HUSOe3Q==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-arm": { + "version": "0.24.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.24.2.tgz", + "integrity": "sha512-n0WRM/gWIdU29J57hJyUdIsk0WarGd6To0s+Y+LwvlC55wt+GT/OgkwoXCXvIue1i1sSNWblHEig00GBWiJgfA==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-arm64": { + "version": "0.24.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.24.2.tgz", + "integrity": "sha512-7HnAD6074BW43YvvUmE/35Id9/NB7BeX5EoNkK9obndmZBUk8xmJJeU7DwmUeN7tkysslb2eSl6CTrYz6oEMQg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-ia32": { + "version": "0.24.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.24.2.tgz", + "integrity": "sha512-sfv0tGPQhcZOgTKO3oBE9xpHuUqguHvSo4jl+wjnKwFpapx+vUDcawbwPNuBIAYdRAvIDBfZVvXprIj3HA+Ugw==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-loong64": { + "version": "0.24.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.24.2.tgz", + "integrity": "sha512-CN9AZr8kEndGooS35ntToZLTQLHEjtVB5n7dl8ZcTZMonJ7CCfStrYhrzF97eAecqVbVJ7APOEe18RPI4KLhwQ==", + "cpu": [ + "loong64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-mips64el": { + "version": "0.24.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.24.2.tgz", + "integrity": "sha512-iMkk7qr/wl3exJATwkISxI7kTcmHKE+BlymIAbHO8xanq/TjHaaVThFF6ipWzPHryoFsesNQJPE/3wFJw4+huw==", + "cpu": [ + "mips64el" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-ppc64": { + "version": "0.24.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.24.2.tgz", + "integrity": "sha512-shsVrgCZ57Vr2L8mm39kO5PPIb+843FStGt7sGGoqiiWYconSxwTiuswC1VJZLCjNiMLAMh34jg4VSEQb+iEbw==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-riscv64": { + "version": "0.24.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.24.2.tgz", + "integrity": "sha512-4eSFWnU9Hhd68fW16GD0TINewo1L6dRrB+oLNNbYyMUAeOD2yCK5KXGK1GH4qD/kT+bTEXjsyTCiJGHPZ3eM9Q==", + "cpu": [ + "riscv64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-s390x": { + "version": "0.24.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.24.2.tgz", + "integrity": "sha512-S0Bh0A53b0YHL2XEXC20bHLuGMOhFDO6GN4b3YjRLK//Ep3ql3erpNcPlEFed93hsQAjAQDNsvcK+hV90FubSw==", + "cpu": [ + "s390x" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-x64": { + "version": "0.24.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.24.2.tgz", + "integrity": "sha512-8Qi4nQcCTbLnK9WoMjdC9NiTG6/E38RNICU6sUNqK0QFxCYgoARqVqxdFmWkdonVsvGqWhmm7MO0jyTqLqwj0Q==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/netbsd-arm64": { + "version": "0.24.2", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-arm64/-/netbsd-arm64-0.24.2.tgz", + "integrity": "sha512-wuLK/VztRRpMt9zyHSazyCVdCXlpHkKm34WUyinD2lzK07FAHTq0KQvZZlXikNWkDGoT6x3TD51jKQ7gMVpopw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/netbsd-x64": { + "version": "0.24.2", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.24.2.tgz", + "integrity": "sha512-VefFaQUc4FMmJuAxmIHgUmfNiLXY438XrL4GDNV1Y1H/RW3qow68xTwjZKfj/+Plp9NANmzbH5R40Meudu8mmw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openbsd-arm64": { + "version": "0.24.2", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-arm64/-/openbsd-arm64-0.24.2.tgz", + "integrity": "sha512-YQbi46SBct6iKnszhSvdluqDmxCJA+Pu280Av9WICNwQmMxV7nLRHZfjQzwbPs3jeWnuAhE9Jy0NrnJ12Oz+0A==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openbsd-x64": { + "version": "0.24.2", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.24.2.tgz", + "integrity": "sha512-+iDS6zpNM6EnJyWv0bMGLWSWeXGN/HTaF/LXHXHwejGsVi+ooqDfMCCTerNFxEkM3wYVcExkeGXNqshc9iMaOA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/sunos-x64": { + "version": "0.24.2", + "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.24.2.tgz", + "integrity": "sha512-hTdsW27jcktEvpwNHJU4ZwWFGkz2zRJUz8pvddmXPtXDzVKTTINmlmga3ZzwcuMpUvLw7JkLy9QLKyGpD2Yxig==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "sunos" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-arm64": { + "version": "0.24.2", + "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.24.2.tgz", + "integrity": "sha512-LihEQ2BBKVFLOC9ZItT9iFprsE9tqjDjnbulhHoFxYQtQfai7qfluVODIYxt1PgdoyQkz23+01rzwNwYfutxUQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-ia32": { + "version": "0.24.2", + "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.24.2.tgz", + "integrity": "sha512-q+iGUwfs8tncmFC9pcnD5IvRHAzmbwQ3GPS5/ceCyHdjXubwQWI12MKWSNSMYLJMq23/IUCvJMS76PDqXe1fxA==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-x64": { + "version": "0.24.2", + "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.24.2.tgz", + "integrity": "sha512-7VTgWzgMGvup6aSqDPLiW5zHaxYJGTO4OokMjIlrCtf+VpEL+cXKtCvg723iguPYI5oaUNdS+/V7OU2gvXVWEg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@marijn/find-cluster-break": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@marijn/find-cluster-break/-/find-cluster-break-1.0.2.tgz", + "integrity": "sha512-l0h88YhZFyKdXIFNfSWpyjStDjGHwZ/U7iobcK1cQQD8sejsONdQtTVU+1wVN1PBw40PiiHB1vA5S7VTfQiP9g==", + "dev": true, + "license": "MIT", + "peer": true + }, + "node_modules/@types/codemirror": { + "version": "5.60.8", + "resolved": "https://registry.npmjs.org/@types/codemirror/-/codemirror-5.60.8.tgz", + "integrity": "sha512-VjFgDF/eB+Aklcy15TtOTLQeMjTo07k7KAjql8OK5Dirr7a6sJY4T1uVBDuTVG9VEmn1uUsohOpYnVfgC6/jyw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/tern": "*" + } + }, + "node_modules/@types/estree": { + "version": "1.0.8", + "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.8.tgz", + "integrity": "sha512-dWHzHa2WqEXI/O1E9OjrocMTKJl2mSrEolh1Iomrv6U+JuNwaHXsXx9bLu5gG7BUWFIN0skIQJQ/L1rIex4X6w==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/node": { + "version": "20.19.39", + "resolved": "https://registry.npmjs.org/@types/node/-/node-20.19.39.tgz", + "integrity": "sha512-orrrD74MBUyK8jOAD/r0+lfa1I2MO6I+vAkmAWzMYbCcgrN4lCrmK52gRFQq/JRxfYPfonkr4b0jcY7Olqdqbw==", + "dev": true, + "license": "MIT", + "dependencies": { + "undici-types": "~6.21.0" + } + }, + "node_modules/@types/tern": { + "version": "0.23.9", + "resolved": "https://registry.npmjs.org/@types/tern/-/tern-0.23.9.tgz", + "integrity": "sha512-ypzHFE/wBzh+BlH6rrBgS5I/Z7RD21pGhZ2rltb/+ZrVM1awdZwjx7hE5XfuYgHWk9uvV5HLZN3SloevCAp3Bw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/estree": "*" + } + }, + "node_modules/crelt": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/crelt/-/crelt-1.0.6.tgz", + "integrity": "sha512-VQ2MBenTq1fWZUH9DJNGti7kKv6EeAuYr3cLwxUWhIu1baTaXh4Ib5W2CqHVqib4/MqbYGJqiL3Zb8GJZr3l4g==", + "dev": true, + "license": "MIT", + "peer": true + }, + "node_modules/esbuild": { + "version": "0.24.2", + "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.24.2.tgz", + "integrity": "sha512-+9egpBW8I3CD5XPe0n6BfT5fxLzxrlDzqydF3aviG+9ni1lDC/OvMHcxqEFV0+LANZG5R1bFMWfUrjVsdwxJvA==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "bin": { + "esbuild": "bin/esbuild" + }, + "engines": { + "node": ">=18" + }, + "optionalDependencies": { + "@esbuild/aix-ppc64": "0.24.2", + "@esbuild/android-arm": "0.24.2", + "@esbuild/android-arm64": "0.24.2", + "@esbuild/android-x64": "0.24.2", + "@esbuild/darwin-arm64": "0.24.2", + "@esbuild/darwin-x64": "0.24.2", + "@esbuild/freebsd-arm64": "0.24.2", + "@esbuild/freebsd-x64": "0.24.2", + "@esbuild/linux-arm": "0.24.2", + "@esbuild/linux-arm64": "0.24.2", + "@esbuild/linux-ia32": "0.24.2", + "@esbuild/linux-loong64": "0.24.2", + "@esbuild/linux-mips64el": "0.24.2", + "@esbuild/linux-ppc64": "0.24.2", + "@esbuild/linux-riscv64": "0.24.2", + "@esbuild/linux-s390x": "0.24.2", + "@esbuild/linux-x64": "0.24.2", + "@esbuild/netbsd-arm64": "0.24.2", + "@esbuild/netbsd-x64": "0.24.2", + "@esbuild/openbsd-arm64": "0.24.2", + "@esbuild/openbsd-x64": "0.24.2", + "@esbuild/sunos-x64": "0.24.2", + "@esbuild/win32-arm64": "0.24.2", + "@esbuild/win32-ia32": "0.24.2", + "@esbuild/win32-x64": "0.24.2" + } + }, + "node_modules/moment": { + "version": "2.29.4", + "resolved": "https://registry.npmjs.org/moment/-/moment-2.29.4.tgz", + "integrity": "sha512-5LC9SOxjSc2HF6vO2CyuTDNivEdoz2IvyJJGj6X8DJ0eFyfszE0QiEd+iXmBvUP3WHxSjFH/vIsA0EN00cgr8w==", + "dev": true, + "license": "MIT", + "engines": { + "node": "*" + } + }, + "node_modules/obsidian": { + "version": "1.12.3", + "resolved": "https://registry.npmjs.org/obsidian/-/obsidian-1.12.3.tgz", + "integrity": "sha512-HxWqe763dOqzXjnNiHmAJTRERN8KILBSqxDSEqbeSr7W8R8Jxezzbca+nz1LiiqXnMpM8lV2jzAezw3CZ4xNUw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/codemirror": "5.60.8", + "moment": "2.29.4" + }, + "peerDependencies": { + "@codemirror/state": "6.5.0", + "@codemirror/view": "6.38.6" + } + }, + "node_modules/style-mod": { + "version": "4.1.3", + "resolved": "https://registry.npmjs.org/style-mod/-/style-mod-4.1.3.tgz", + "integrity": "sha512-i/n8VsZydrugj3Iuzll8+x/00GH2vnYsk1eomD8QiRrSAeW6ItbCQDtfXCeJHd0iwiNagqjQkvpvREEPtW3IoQ==", + "dev": true, + "license": "MIT", + "peer": true + }, + "node_modules/tslib": { + "version": "2.8.1", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", + "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==", + "dev": true, + "license": "0BSD" + }, + "node_modules/typescript": { + "version": "5.9.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz", + "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=14.17" + } + }, + "node_modules/undici-types": { + "version": "6.21.0", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.21.0.tgz", + "integrity": "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/w3c-keyname": { + "version": "2.2.8", + "resolved": "https://registry.npmjs.org/w3c-keyname/-/w3c-keyname-2.2.8.tgz", + "integrity": "sha512-dpojBhNsCNN7T82Tm7k26A6G9ML3NkhDsnw9n/eoxSRlVBB4CEtIQ/KTCLI2Fwf3ataSXRhYFkQi3SlnFwPvPQ==", + "dev": true, + "license": "MIT", + "peer": true + } + } +} diff --git a/package.json b/package.json new file mode 100644 index 0000000..8121175 --- /dev/null +++ b/package.json @@ -0,0 +1,26 @@ +{ + "name": "obsidian-vault-sync", + "version": "0.1.0", + "description": "Sync Obsidian vault with GitHub via REST API. Works on iOS for any vault size including images.", + "main": "main.js", + "scripts": { + "dev": "node esbuild.config.mjs", + "build": "node esbuild.config.mjs production" + }, + "keywords": [ + "obsidian", + "obsidian-plugin", + "github", + "sync", + "ios" + ], + "author": "Andrew Boldi", + "license": "MIT", + "devDependencies": { + "@types/node": "^20.11.0", + "esbuild": "^0.24.2", + "obsidian": "^1.4.0", + "tslib": "^2.6.2", + "typescript": "^5.3.3" + } +} diff --git a/src/github.ts b/src/github.ts new file mode 100644 index 0000000..6f22020 --- /dev/null +++ b/src/github.ts @@ -0,0 +1,54 @@ +// GitHub REST helpers for Vault Sync. +// Phase 1 only needs: parse repo URL, fetch latest branch SHA, build tarball URL. + +export interface RepoRef { + owner: string; + repo: string; +} + +export const USER_AGENT = "obsidian-vault-sync"; +export const API_BASE = "https://api.github.com"; + +export function authHeaders(pat: string): Record { + return { + Authorization: `token ${pat}`, + "User-Agent": USER_AGENT, + Accept: "application/vnd.github+json", + "X-GitHub-Api-Version": "2022-11-28", + }; +} + +export function parseRepoUrl(url: string): RepoRef { + if (!url) throw new Error("Repository URL is empty"); + let cleaned = url.trim().replace(/\.git$/, "").replace(/\/+$/, ""); + // Allow either full URL or "owner/repo". + let m = cleaned.match(/^https?:\/\/github\.com\/([^/]+)\/([^/]+)$/i); + if (!m) m = cleaned.match(/^([^/\s]+)\/([^/\s]+)$/); + if (!m) throw new Error(`Cannot parse repo URL: ${url}`); + return { owner: m[1], repo: m[2] }; +} + +export async function getBranchSha( + ref: RepoRef, + branch: string, + pat: string +): Promise { + const url = `${API_BASE}/repos/${ref.owner}/${ref.repo}/branches/${encodeURIComponent( + branch + )}`; + const res = await fetch(url, { headers: authHeaders(pat) }); + if (!res.ok) { + const body = await res.text(); + throw new Error( + `GitHub branch lookup failed: ${res.status} ${res.statusText} — ${body.slice(0, 300)}` + ); + } + const data = (await res.json()) as { commit?: { sha?: string } }; + const sha = data?.commit?.sha; + if (!sha) throw new Error("GitHub response missing commit.sha"); + return sha; +} + +export function tarballUrl(ref: RepoRef, sha: string): string { + return `${API_BASE}/repos/${ref.owner}/${ref.repo}/tarball/${sha}`; +} diff --git a/src/main.ts b/src/main.ts new file mode 100644 index 0000000..4744c5b --- /dev/null +++ b/src/main.ts @@ -0,0 +1,39 @@ +import { Notice, Plugin } from "obsidian"; +import { + DEFAULT_SETTINGS, + VaultSyncSettingTab, + type VaultSyncSettings, +} from "./settings"; +import { seedFromGitHub } from "./seed"; + +export default class VaultSyncPlugin extends Plugin { + settings!: VaultSyncSettings; + + async onload(): Promise { + await this.loadSettings(); + this.addSettingTab(new VaultSyncSettingTab(this.app, this)); + + this.addCommand({ + id: "seed-from-github", + name: "Seed from GitHub", + callback: async () => { + try { + await seedFromGitHub(this); + } catch (e) { + const msg = e instanceof Error ? e.message : String(e); + console.error("[Vault Sync] seed failed", e); + new Notice(`Vault Sync seed failed: ${msg}`, 10000); + } + }, + }); + } + + async loadSettings(): Promise { + const data = (await this.loadData()) ?? {}; + this.settings = Object.assign({}, DEFAULT_SETTINGS, data); + } + + async saveSettings(): Promise { + await this.saveData(this.settings); + } +} diff --git a/src/seed.ts b/src/seed.ts new file mode 100644 index 0000000..a922c35 --- /dev/null +++ b/src/seed.ts @@ -0,0 +1,141 @@ +import { Notice } from "obsidian"; +import type VaultSyncPlugin from "./main"; +import { + authHeaders, + getBranchSha, + parseRepoUrl, + tarballUrl, +} from "./github"; +import { parseTar } from "./tar"; + +function formatBytes(n: number): string { + if (n < 1024) return `${n} B`; + if (n < 1024 * 1024) return `${(n / 1024).toFixed(1)} KB`; + if (n < 1024 * 1024 * 1024) return `${(n / (1024 * 1024)).toFixed(1)} MB`; + return `${(n / (1024 * 1024 * 1024)).toFixed(2)} GB`; +} + +/** + * Strip GitHub's wrapper directory ("{owner}-{repo}-{shortsha}/") from a tar entry path. + * Returns null if the result would be empty (i.e. the wrapper dir itself). + */ +function stripWrapperDir(path: string): string | null { + const normalized = path.replace(/\\/g, "/"); + const idx = normalized.indexOf("/"); + if (idx < 0) return null; // wrapper dir entry alone + const rest = normalized.slice(idx + 1); + if (!rest || rest === "/") return null; + // Defense in depth: refuse path traversal. + if (rest.startsWith("/") || rest.includes("../")) return null; + return rest; +} + +async function ensureDir(plugin: VaultSyncPlugin, dirPath: string): Promise { + if (!dirPath || dirPath === "/" || dirPath === ".") return; + try { + await plugin.app.vault.adapter.mkdir(dirPath); + } catch (_e) { + // Already exists, or parent created concurrently — ignore. + } +} + +async function ensureParentDirs( + plugin: VaultSyncPlugin, + filePath: string +): Promise { + const parts = filePath.split("/"); + parts.pop(); // remove file name + if (parts.length === 0) return; + let acc = ""; + for (const p of parts) { + if (!p) continue; + acc = acc ? `${acc}/${p}` : p; + await ensureDir(plugin, acc); + } +} + +export async function seedFromGitHub(plugin: VaultSyncPlugin): Promise { + const { pat, repoUrl, branch } = plugin.settings; + if (!pat) throw new Error("Set a GitHub PAT in Vault Sync settings."); + if (!repoUrl) throw new Error("Set a repo URL in Vault Sync settings."); + + const ref = parseRepoUrl(repoUrl); + const notice = new Notice("Vault Sync: resolving branch…", 0); + + let sha: string; + try { + sha = await getBranchSha(ref, branch || "main", pat); + } catch (e) { + notice.hide(); + throw e; + } + + notice.setMessage(`Vault Sync: downloading ${ref.owner}/${ref.repo}@${sha.slice(0, 7)}…`); + + const url = tarballUrl(ref, sha); + const res = await fetch(url, { headers: authHeaders(pat) }); + if (!res.ok || !res.body) { + notice.hide(); + const body = res.body ? await res.text().catch(() => "") : ""; + throw new Error( + `Tarball fetch failed: ${res.status} ${res.statusText} — ${body.slice(0, 300)}` + ); + } + + // GitHub serves tarballs with Content-Encoding: gzip already applied to a + // *tar.gz* payload. fetch() automatically decompresses transport-level + // Content-Encoding, but the body itself is still a gzipped tar — we must + // pipe through DecompressionStream('gzip') ourselves. + let tarStream: ReadableStream; + try { + tarStream = res.body.pipeThrough(new DecompressionStream("gzip")); + } catch (e) { + notice.hide(); + throw new Error( + "DecompressionStream('gzip') is not available in this WebView. " + + "Vault Sync requires iOS 16.4+/modern Chromium. " + + String(e) + ); + } + + let fileCount = 0; + let byteCount = 0; + let lastUiUpdate = 0; + + for await (const entry of parseTar(tarStream)) { + const stripped = stripWrapperDir(entry.name); + if (!stripped) continue; + + if (entry.type === "dir") { + await ensureDir(plugin, stripped.replace(/\/$/, "")); + continue; + } + + // File. + await ensureParentDirs(plugin, stripped); + // Convert Uint8Array view -> standalone ArrayBuffer for writeBinary. + const ab = entry.content.buffer.slice( + entry.content.byteOffset, + entry.content.byteOffset + entry.content.byteLength + ) as ArrayBuffer; + await plugin.app.vault.adapter.writeBinary(stripped, ab); + + fileCount++; + byteCount += entry.content.byteLength; + + if (fileCount - lastUiUpdate >= 25) { + notice.setMessage( + `Vault Sync: extracted ${fileCount} files, ${formatBytes(byteCount)}` + ); + lastUiUpdate = fileCount; + } + } + + plugin.settings.lastCommitSha = sha; + await plugin.saveSettings(); + + notice.setMessage( + `Vault Sync: seed complete — ${fileCount} files, ${formatBytes(byteCount)} @ ${sha.slice(0, 7)}` + ); + setTimeout(() => notice.hide(), 6000); +} diff --git a/src/settings.ts b/src/settings.ts new file mode 100644 index 0000000..7dddda9 --- /dev/null +++ b/src/settings.ts @@ -0,0 +1,83 @@ +import { App, PluginSettingTab, Setting } from "obsidian"; +import type VaultSyncPlugin from "./main"; + +export interface VaultSyncSettings { + pat: string; + repoUrl: string; + branch: string; + lastCommitSha: string; +} + +export const DEFAULT_SETTINGS: VaultSyncSettings = { + pat: "", + repoUrl: "", + branch: "main", + lastCommitSha: "", +}; + +export class VaultSyncSettingTab extends PluginSettingTab { + plugin: VaultSyncPlugin; + + constructor(app: App, plugin: VaultSyncPlugin) { + super(app, plugin); + this.plugin = plugin; + } + + display(): void { + const { containerEl } = this; + containerEl.empty(); + + containerEl.createEl("h2", { text: "Vault Sync (REST)" }); + + new Setting(containerEl) + .setName("GitHub Personal Access Token") + .setDesc( + "Fine-grained or classic PAT with repo (contents: read/write) scope on the target repo." + ) + .addText((text) => { + text.inputEl.type = "password"; + text.setPlaceholder("ghp_…") + .setValue(this.plugin.settings.pat) + .onChange(async (value) => { + this.plugin.settings.pat = value.trim(); + await this.plugin.saveSettings(); + }); + }); + + new Setting(containerEl) + .setName("Repository URL") + .setDesc("e.g. https://github.com/owner/repo") + .addText((text) => + text + .setPlaceholder("https://github.com/owner/repo") + .setValue(this.plugin.settings.repoUrl) + .onChange(async (value) => { + this.plugin.settings.repoUrl = value.trim(); + await this.plugin.saveSettings(); + }) + ); + + new Setting(containerEl) + .setName("Branch") + .setDesc("Branch to sync (default: main)") + .addText((text) => + text + .setPlaceholder("main") + .setValue(this.plugin.settings.branch) + .onChange(async (value) => { + this.plugin.settings.branch = value.trim() || "main"; + await this.plugin.saveSettings(); + }) + ); + + new Setting(containerEl) + .setName("Last commit SHA") + .setDesc( + "Set after a successful seed/pull. Read-only — used as the local sync base." + ) + .addText((text) => { + text.setValue(this.plugin.settings.lastCommitSha || "(none)") + .setDisabled(true); + }); + } +} diff --git a/src/tar.ts b/src/tar.ts new file mode 100644 index 0000000..da160fb --- /dev/null +++ b/src/tar.ts @@ -0,0 +1,210 @@ +// Minimal streaming tar parser for the POSIX ustar format produced by GitHub. +// Yields { name, type, content } entries from a ReadableStream. +// No dependencies. Browser-safe. +// +// Tar layout (USTAR): +// bytes 0–100 filename (NUL-terminated, may overflow into prefix) +// bytes 100–108 mode +// bytes 108–116 uid +// bytes 116–124 gid +// bytes 124–136 size in octal ASCII (NUL-terminated) +// bytes 136–148 mtime +// bytes 148–156 checksum +// byte 156 typeflag ('0' or '\0' = file, '5' = dir, others ignored v1) +// bytes 157–257 linkname +// bytes 257–263 ustar magic ("ustar\0") +// bytes 265–297 uname +// bytes 297–329 gname +// bytes 329–337 devmajor +// bytes 337–345 devminor +// bytes 345–500 prefix (prepended to name with "/" if non-empty) +// Header is 512 bytes; data follows, padded up to a 512-byte boundary. +// End of archive = two consecutive 512-byte zero blocks. + +export interface TarEntry { + name: string; + type: "file" | "dir"; + content: Uint8Array; // empty for dir +} + +const BLOCK = 512; + +function readCString(buf: Uint8Array, off: number, len: number): string { + let end = off; + const lim = off + len; + while (end < lim && buf[end] !== 0) end++; + return new TextDecoder("utf-8").decode(buf.subarray(off, end)); +} + +function parseOctal(buf: Uint8Array, off: number, len: number): number { + // Trim trailing NUL/space; ignore leading spaces. + let s = ""; + for (let i = off; i < off + len; i++) { + const c = buf[i]; + if (c === 0 || c === 0x20) { + if (s.length > 0) break; + continue; + } + s += String.fromCharCode(c); + } + if (!s) return 0; + return parseInt(s, 8) | 0; +} + +function isAllZero(buf: Uint8Array, off: number, len: number): boolean { + for (let i = off; i < off + len; i++) if (buf[i] !== 0) return false; + return true; +} + +/** + * A small append-only buffer that lets us peek at the head and shift consumed + * bytes without copying everything every time. We grow by doubling and shift + * down only when the consumed prefix exceeds half the buffer. + */ +class ChunkBuffer { + private buf: Uint8Array = new Uint8Array(0); + private start = 0; // first valid byte + private end = 0; // one past last valid byte + + get length(): number { + return this.end - this.start; + } + + push(chunk: Uint8Array): void { + const need = this.length + chunk.length; + if (need > this.buf.length - this.start) { + // Either grow or shift to make room. + if (this.start > 0 && need <= this.buf.length) { + this.buf.copyWithin(0, this.start, this.end); + this.end -= this.start; + this.start = 0; + } else { + let cap = Math.max(this.buf.length * 2, 64 * 1024); + while (cap < need) cap *= 2; + const next = new Uint8Array(cap); + next.set(this.buf.subarray(this.start, this.end), 0); + this.end -= this.start; + this.start = 0; + this.buf = next; + } + } + this.buf.set(chunk, this.end); + this.end += chunk.length; + } + + /** Returns a view of the next `n` bytes without consuming. Caller must ensure length >= n. */ + peek(n: number): Uint8Array { + return this.buf.subarray(this.start, this.start + n); + } + + /** Consume `n` bytes, returning a copy (safe to keep). */ + take(n: number): Uint8Array { + const out = this.buf.slice(this.start, this.start + n); + this.start += n; + if (this.start > this.buf.length / 2) { + // Compact occasionally to bound memory. + this.buf.copyWithin(0, this.start, this.end); + this.end -= this.start; + this.start = 0; + } + return out; + } + + /** Discard `n` bytes (e.g. padding). */ + skip(n: number): void { + this.start += n; + if (this.start > this.buf.length / 2) { + this.buf.copyWithin(0, this.start, this.end); + this.end -= this.start; + this.start = 0; + } + } +} + +/** + * Parse a tar stream and yield entries. The stream is assumed uncompressed + * (gzip decompression must happen upstream via DecompressionStream). + */ +export async function* parseTar( + stream: ReadableStream +): AsyncGenerator { + const reader = stream.getReader(); + const buf = new ChunkBuffer(); + let zeroBlocks = 0; + let done = false; + + async function fillTo(n: number): Promise { + while (buf.length < n) { + if (done) return false; + const { value, done: d } = await reader.read(); + if (d) { + done = true; + return buf.length >= n; + } + if (value && value.length) buf.push(value); + } + return true; + } + + while (true) { + if (!(await fillTo(BLOCK))) break; + + const header = buf.peek(BLOCK); + if (isAllZero(header, 0, BLOCK)) { + buf.skip(BLOCK); + zeroBlocks++; + if (zeroBlocks >= 2) break; // end-of-archive marker + continue; + } + zeroBlocks = 0; + + const namePart = readCString(header, 0, 100); + const size = parseOctal(header, 124, 12); + const typeflagByte = header[156]; + const typeflag = typeflagByte === 0 ? "0" : String.fromCharCode(typeflagByte); + + // ustar prefix for long names. + const ustar = readCString(header, 257, 6); + let prefix = ""; + if (ustar === "ustar" || ustar.startsWith("ustar")) { + prefix = readCString(header, 345, 155); + } + const name = prefix ? `${prefix}/${namePart}` : namePart; + + buf.skip(BLOCK); // header consumed + + const padded = size > 0 ? Math.ceil(size / BLOCK) * BLOCK : 0; + + // Skip non-file/dir entries (long-link records, symlinks, etc.). + if (typeflag !== "0" && typeflag !== "5" && typeflag !== "\0") { + // Drain the data + padding without keeping it. + let remaining = padded; + while (remaining > 0) { + if (!(await fillTo(Math.min(remaining, BLOCK)))) { + return; // truncated + } + const take = Math.min(remaining, buf.length); + buf.skip(take); + remaining -= take; + } + continue; + } + + if (typeflag === "5") { + // Directory entry — no data, but padded is 0 anyway. + yield { name, type: "dir", content: new Uint8Array(0) }; + continue; + } + + // File entry: read `size` bytes (and skip padding to next block boundary). + if (!(await fillTo(padded))) { + // Truncated file — best-effort: yield what we have, then stop. + const have = Math.min(buf.length, size); + yield { name, type: "file", content: buf.take(have) }; + return; + } + const content = buf.take(size); + if (padded > size) buf.skip(padded - size); + yield { name, type: "file", content }; + } +} diff --git a/styles.css b/styles.css new file mode 100644 index 0000000..a117e16 --- /dev/null +++ b/styles.css @@ -0,0 +1 @@ +/* Vault Sync (REST) styles - placeholder for Phase 1 */ diff --git a/tsconfig.json b/tsconfig.json new file mode 100644 index 0000000..77d1438 --- /dev/null +++ b/tsconfig.json @@ -0,0 +1,26 @@ +{ + "compilerOptions": { + "baseUrl": ".", + "inlineSourceMap": true, + "inlineSources": true, + "module": "ESNext", + "target": "ES2020", + "allowJs": true, + "noImplicitAny": true, + "strict": true, + "isolatedModules": true, + "moduleResolution": "node", + "importHelpers": true, + "allowSyntheticDefaultImports": true, + "esModuleInterop": true, + "skipLibCheck": true, + "lib": [ + "DOM", + "ES2020", + "ES2021.WeakRef" + ] + }, + "include": [ + "src/**/*.ts" + ] +}