mirror of
https://github.com/gapmiss/substack-clipper.git
synced 2026-07-22 07:48:24 +00:00
Initial commit: fetch, parse, and archive Substack posts as Markdown with media downloads and threaded comments
This commit is contained in:
commit
291fd7919c
24 changed files with 7068 additions and 0 deletions
49
.github/workflows/release.yml
vendored
Normal file
49
.github/workflows/release.yml
vendored
Normal file
|
|
@ -0,0 +1,49 @@
|
|||
name: Release
|
||||
|
||||
on:
|
||||
push:
|
||||
tags:
|
||||
- '*.*.*'
|
||||
|
||||
permissions:
|
||||
contents: write
|
||||
id-token: write
|
||||
attestations: write
|
||||
|
||||
jobs:
|
||||
release:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Setup Node.js
|
||||
uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: '20'
|
||||
cache: 'npm'
|
||||
|
||||
- name: Install dependencies
|
||||
run: npm ci
|
||||
|
||||
- name: Build
|
||||
run: npm run build
|
||||
|
||||
- name: Attest main.js
|
||||
uses: actions/attest-build-provenance@v2
|
||||
with:
|
||||
subject-path: main.js
|
||||
|
||||
- name: Attest styles.css
|
||||
uses: actions/attest-build-provenance@v2
|
||||
with:
|
||||
subject-path: styles.css
|
||||
|
||||
- name: Create Release
|
||||
uses: softprops/action-gh-release@v2
|
||||
with:
|
||||
files: |
|
||||
main.js
|
||||
manifest.json
|
||||
styles.css
|
||||
generate_release_notes: true
|
||||
29
.gitignore
vendored
Normal file
29
.gitignore
vendored
Normal file
|
|
@ -0,0 +1,29 @@
|
|||
# Logs
|
||||
*.log
|
||||
npm-debug.log*
|
||||
|
||||
# Dependency directories
|
||||
node_modules/
|
||||
|
||||
# Build output
|
||||
main.js
|
||||
*.js.map
|
||||
|
||||
# OS files
|
||||
.DS_Store
|
||||
Thumbs.db
|
||||
|
||||
# IDE
|
||||
.vscode/
|
||||
.idea/
|
||||
|
||||
# TypeScript cache
|
||||
*.tsbuildinfo
|
||||
|
||||
# Privacy
|
||||
data.json
|
||||
.env
|
||||
|
||||
# Agent settings (local)
|
||||
.claude
|
||||
CLAUDE.md
|
||||
59
CONTRIBUTING.md
Normal file
59
CONTRIBUTING.md
Normal file
|
|
@ -0,0 +1,59 @@
|
|||
# Contributing
|
||||
|
||||
Thanks for your interest in contributing to Substack Clipper.
|
||||
|
||||
## Development setup
|
||||
|
||||
```bash
|
||||
git clone https://github.com/gapmiss/substack-clipper.git
|
||||
cd substack-clipper
|
||||
npm install
|
||||
```
|
||||
|
||||
### Build commands
|
||||
|
||||
| Command | Description |
|
||||
|---------|-------------|
|
||||
| `npm run build` | Typecheck + production build |
|
||||
| `npm run dev` | Watch mode (rebuilds on file change) |
|
||||
| `npm run lint` | Run ESLint |
|
||||
|
||||
### Testing in Obsidian
|
||||
|
||||
1. Run `npm run dev` for watch mode.
|
||||
2. Symlink or copy `main.js`, `manifest.json`, and `styles.css` to your test vault at `.obsidian/plugins/substack-clipper/`.
|
||||
3. Enable the plugin in Obsidian settings.
|
||||
4. Use `Ctrl/Cmd + P` > "Clip substack post" to test.
|
||||
|
||||
## Code standards
|
||||
|
||||
- **TypeScript strict mode** with `noImplicitAny` and `strictNullChecks`.
|
||||
- **ESLint must pass with zero errors and zero warnings** before submitting a PR. Run `npm run lint` to check.
|
||||
- The project uses [`eslint-plugin-obsidianmd`](https://github.com/phibr0/eslint-plugin-obsidianmd) and [`typescript-eslint/recommendedTypeChecked`](https://typescript-eslint.io/getting-started/typed-linting/).
|
||||
- Use `requestUrl()` for all network requests, never `fetch()`.
|
||||
- Use `normalizePath()` for all vault paths.
|
||||
- No regex lookbehind (iOS compatibility).
|
||||
- All UI text must use sentence case ("Clip substack post", not "Clip Substack Post").
|
||||
- No `console.log` — use `console.warn`, `console.error`, or `console.debug` only.
|
||||
- Use `Array.from()` when iterating `NodeListOf<>` collections.
|
||||
|
||||
## Pull requests
|
||||
|
||||
1. Fork the repo and create a feature branch from `main`.
|
||||
2. Make your changes.
|
||||
3. Ensure `npm run build` and `npm run lint` both pass cleanly.
|
||||
4. Test the change in Obsidian with at least one public Substack post.
|
||||
5. Open a PR with a clear description of what changed and why.
|
||||
|
||||
## Reporting issues
|
||||
|
||||
Open an issue on GitHub with:
|
||||
|
||||
- The Substack URL that caused the problem (if applicable).
|
||||
- What you expected to happen.
|
||||
- What actually happened (error messages, screenshots, etc.).
|
||||
- Your Obsidian version and OS.
|
||||
|
||||
## License
|
||||
|
||||
By contributing, you agree that your contributions will be licensed under the [MIT License](LICENSE).
|
||||
21
LICENSE
Normal file
21
LICENSE
Normal file
|
|
@ -0,0 +1,21 @@
|
|||
MIT License
|
||||
|
||||
Copyright (c) 2026 @gapmiss
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all
|
||||
copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
SOFTWARE.
|
||||
74
README.md
Normal file
74
README.md
Normal file
|
|
@ -0,0 +1,74 @@
|
|||
# Substack Clipper
|
||||
|
||||
Archive Substack posts as Obsidian-flavored Markdown with images, media, and threaded comments.
|
||||
|
||||
## Features
|
||||
|
||||
- Saves any public Substack post as a Markdown note with YAML frontmatter
|
||||
- Downloads images, videos, audio, podcast episodes, transcripts, and PDF attachments
|
||||
- Fetches and renders threaded comments as a separate embeddable note
|
||||
- Media files use `![[wikilink]]` embeds for native Obsidian playback
|
||||
- Converts Substack footnotes to standard `[^N]` Markdown footnotes
|
||||
- Strips CDN URL prefixes for clean image references
|
||||
- Handles paywalled posts gracefully (saves the available preview)
|
||||
- Works on desktop and mobile
|
||||
|
||||
## Installation
|
||||
|
||||
### From community plugins
|
||||
|
||||
Search for **Substack Clipper** in Obsidian's community plugin browser.
|
||||
|
||||
### Manual
|
||||
|
||||
1. Download `main.js`, `manifest.json`, and `styles.css` from the [latest release](../../releases/latest).
|
||||
2. Create a folder at `{your-vault}/.obsidian/plugins/substack-clipper/`.
|
||||
3. Copy the three files into that folder.
|
||||
4. Reload Obsidian and enable the plugin in Settings > Community plugins.
|
||||
|
||||
## Usage
|
||||
|
||||
1. Open the command palette (`Ctrl/Cmd + P`).
|
||||
2. Run **Clip substack post**.
|
||||
3. Paste a Substack post URL (must contain `/p/` in the path).
|
||||
4. The plugin fetches the post, downloads media, and creates the note.
|
||||
|
||||
### Output structure
|
||||
|
||||
```
|
||||
Substacks/
|
||||
alice/
|
||||
my-article.md
|
||||
my-article-comments.md
|
||||
my-article/
|
||||
image.jpg
|
||||
video.mp4
|
||||
podcast.mp3
|
||||
attachment.pdf
|
||||
```
|
||||
|
||||
The main note includes frontmatter with title, subtitle, type, audience, date, comment count, and links to all media. Comments are embedded via `![[slug-comments]]`.
|
||||
|
||||
## Settings
|
||||
|
||||
| Setting | Default | Description |
|
||||
|---------|---------|-------------|
|
||||
| Save directory | `Substacks` | Vault-relative folder for saved posts |
|
||||
| Download media | On | Download videos, audio, podcasts, and transcripts (images and attachments always download) |
|
||||
| Download comments | On | Fetch and save threaded comments |
|
||||
| Comment sort order | Most recent first | Sort order for comments (most recent, oldest, or best) |
|
||||
| Save raw JSON | Off | Save the raw Substack API JSON |
|
||||
| Save raw HTML | Off | Save the raw article HTML |
|
||||
|
||||
## Building from source
|
||||
|
||||
```bash
|
||||
git clone https://github.com/gapmiss/substack-clipper.git
|
||||
cd substack-clipper
|
||||
npm install
|
||||
npm run build
|
||||
```
|
||||
|
||||
## License
|
||||
|
||||
[MIT](LICENSE)
|
||||
48
esbuild.config.mjs
Normal file
48
esbuild.config.mjs
Normal file
|
|
@ -0,0 +1,48 @@
|
|||
import esbuild from "esbuild";
|
||||
import process from "process";
|
||||
import { builtinModules } from "node:module";
|
||||
|
||||
const banner =
|
||||
`/*
|
||||
THIS IS A GENERATED/BUNDLED FILE BY ESBUILD
|
||||
if you want to view the source, please visit the github repository of this plugin
|
||||
*/
|
||||
`;
|
||||
|
||||
const prod = (process.argv[2] === "production");
|
||||
|
||||
const context = await esbuild.context({
|
||||
banner: {
|
||||
js: banner,
|
||||
},
|
||||
entryPoints: ["src/main.ts"],
|
||||
bundle: true,
|
||||
external: [
|
||||
"obsidian",
|
||||
"electron",
|
||||
"@codemirror/autocomplete",
|
||||
"@codemirror/collab",
|
||||
"@codemirror/commands",
|
||||
"@codemirror/language",
|
||||
"@codemirror/lint",
|
||||
"@codemirror/search",
|
||||
"@codemirror/state",
|
||||
"@codemirror/view",
|
||||
"@lezer/common",
|
||||
"@lezer/highlight",
|
||||
"@lezer/lr",
|
||||
...builtinModules],
|
||||
format: "cjs",
|
||||
target: "es2018",
|
||||
logLevel: "info",
|
||||
sourcemap: prod ? false : "inline",
|
||||
treeShaking: true,
|
||||
outfile: "main.js",
|
||||
});
|
||||
|
||||
if (prod) {
|
||||
await context.rebuild();
|
||||
process.exit(0);
|
||||
} else {
|
||||
await context.watch();
|
||||
}
|
||||
34
eslint.config.mjs
Normal file
34
eslint.config.mjs
Normal file
|
|
@ -0,0 +1,34 @@
|
|||
import tsParser from "@typescript-eslint/parser";
|
||||
import tseslint from "typescript-eslint";
|
||||
import obsidianmd from "eslint-plugin-obsidianmd";
|
||||
|
||||
export default [
|
||||
{ ignores: ["node_modules/**", "main.js", "*.mjs", "package.json", "package-lock.json", "versions.json", "tsconfig.json"] },
|
||||
// TypeScript-ESLint recommended rules WITH type checking
|
||||
...tseslint.configs.recommendedTypeChecked.map(config => ({
|
||||
...config,
|
||||
files: ["src/**/*.ts"],
|
||||
})),
|
||||
// Obsidian plugin rules (v0.3.0+)
|
||||
...obsidianmd.configs.recommended,
|
||||
// Project-specific config
|
||||
{
|
||||
files: ["src/**/*.ts"],
|
||||
languageOptions: {
|
||||
parser: tsParser,
|
||||
parserOptions: {
|
||||
project: "./tsconfig.json",
|
||||
sourceType: "module",
|
||||
},
|
||||
},
|
||||
rules: {
|
||||
// Console: scanner allows warn, error, debug only
|
||||
"no-console": ["error", { allow: ["warn", "error", "debug"] }],
|
||||
// Allow underscore-prefixed unused params
|
||||
"@typescript-eslint/no-unused-vars": ["error", {
|
||||
argsIgnorePattern: "^_",
|
||||
varsIgnorePattern: "^_",
|
||||
}],
|
||||
},
|
||||
},
|
||||
];
|
||||
10
manifest.json
Normal file
10
manifest.json
Normal file
|
|
@ -0,0 +1,10 @@
|
|||
{
|
||||
"id": "substack-clipper",
|
||||
"name": "Substack Clipper",
|
||||
"version": "0.1.0",
|
||||
"minAppVersion": "1.13.0",
|
||||
"description": "Archive Substack posts as Markdown with images, media, and comments.",
|
||||
"author": "@gapmiss",
|
||||
"authorUrl": "https://github.com/gapmiss",
|
||||
"isDesktopOnly": false
|
||||
}
|
||||
5222
package-lock.json
generated
Normal file
5222
package-lock.json
generated
Normal file
File diff suppressed because it is too large
Load diff
40
package.json
Normal file
40
package.json
Normal file
|
|
@ -0,0 +1,40 @@
|
|||
{
|
||||
"name": "substack-clipper",
|
||||
"version": "0.1.0",
|
||||
"description": "Clip substack articles",
|
||||
"main": "main.js",
|
||||
"scripts": {
|
||||
"dev": "node esbuild.config.mjs",
|
||||
"build": "tsc -noEmit -skipLibCheck && node esbuild.config.mjs production",
|
||||
"lint": "eslint src/",
|
||||
"version": "node version-bump.mjs && git add manifest.json versions.json"
|
||||
},
|
||||
"keywords": [
|
||||
"obsidian",
|
||||
"obsidian-plugin"
|
||||
],
|
||||
"author": "@gapmiss",
|
||||
"license": "MIT",
|
||||
"devDependencies": {
|
||||
"@eslint/js": "^9.30.1",
|
||||
"@types/node": "^16.11.6",
|
||||
"@types/turndown": "^5.0.6",
|
||||
"@typescript-eslint/parser": "^8.35.1",
|
||||
"esbuild": "^0.28.1",
|
||||
"eslint": "^9.30.1",
|
||||
"eslint-plugin-no-unsanitized": "^4.1.2",
|
||||
"eslint-plugin-obsidianmd": "^0.3.0",
|
||||
"globals": "^14.0.0",
|
||||
"jiti": "^2.6.1",
|
||||
"tslib": "^2.4.0",
|
||||
"typescript": "^5.8.2",
|
||||
"typescript-eslint": "^8.35.1"
|
||||
},
|
||||
"dependencies": {
|
||||
"obsidian": "latest",
|
||||
"turndown": "^7.2.4"
|
||||
},
|
||||
"allowScripts": {
|
||||
"esbuild@0.28.1": true
|
||||
}
|
||||
}
|
||||
332
release.mjs
Normal file
332
release.mjs
Normal file
|
|
@ -0,0 +1,332 @@
|
|||
import { readFileSync, writeFileSync } from 'fs';
|
||||
import { execSync } from 'child_process';
|
||||
import path from 'path';
|
||||
|
||||
/**
|
||||
* Version Update and Deployment Automation Script
|
||||
*
|
||||
* 1. Update version in package.json
|
||||
* 2. Synchronize version in manifest.json, versions.json
|
||||
* 3. Run build
|
||||
* 4. Create Git commit
|
||||
* 5. Create Git tag
|
||||
* 6. Push changes to GitHub (REQUIRES Github CLI: https://cli.github.com)
|
||||
*/
|
||||
|
||||
// Version type definitions
|
||||
const VERSION_TYPES = {
|
||||
PATCH: 'patch',
|
||||
MINOR: 'minor',
|
||||
MAJOR: 'major'
|
||||
};
|
||||
|
||||
// Default settings
|
||||
const DEFAULT_VERSION_TYPE = VERSION_TYPES.PATCH;
|
||||
const RELEASE_FILES = ['main.js', 'manifest.json', 'styles.css'];
|
||||
const MIN_APP_VERSION = JSON.parse(readFileSync(path.resolve(process.cwd(), 'manifest.json'), 'utf8')).minAppVersion;
|
||||
|
||||
// Store original versions for rollback if needed
|
||||
let originalPackageVersion = '';
|
||||
let originalManifestVersion = '';
|
||||
|
||||
/**
|
||||
* Updates version value from the version string.
|
||||
* @param {string} version - Current version (e.g., '0.2.2')
|
||||
* @param {string} type - Update type ('patch', 'minor', 'major')
|
||||
* @returns {string} Updated version
|
||||
*/
|
||||
const updateVersion = (version, type = DEFAULT_VERSION_TYPE) => {
|
||||
const [major, minor, patch] = version.split('.').map(Number);
|
||||
|
||||
switch (type) {
|
||||
case VERSION_TYPES.MAJOR:
|
||||
return `${major + 1}.0.0`;
|
||||
case VERSION_TYPES.MINOR:
|
||||
return `${major}.${minor + 1}.0`;
|
||||
case VERSION_TYPES.PATCH:
|
||||
default:
|
||||
return `${major}.${minor}.${patch + 1}`;
|
||||
}
|
||||
};
|
||||
|
||||
/**
|
||||
* Gets version information before any changes are made.
|
||||
* @returns {string} Previous version
|
||||
*/
|
||||
const getPreviousVersion = () => {
|
||||
const manifestPath = path.resolve(process.cwd(), 'manifest.json');
|
||||
const manifestData = JSON.parse(readFileSync(manifestPath, 'utf8'));
|
||||
return manifestData.version;
|
||||
}
|
||||
|
||||
/**
|
||||
* Updates version information in package.json file.
|
||||
* @param {string} versionType - Version type to update
|
||||
* @returns {string} New version
|
||||
*/
|
||||
const updatePackageVersion = (versionType) => {
|
||||
const packagePath = path.resolve(process.cwd(), 'package.json');
|
||||
const packageData = JSON.parse(readFileSync(packagePath, 'utf8'));
|
||||
|
||||
const currentVersion = packageData.version;
|
||||
originalPackageVersion = currentVersion; // Store original version for possible rollback
|
||||
const newVersion = updateVersion(currentVersion, versionType);
|
||||
|
||||
packageData.version = newVersion;
|
||||
writeFileSync(packagePath, JSON.stringify(packageData, null, '\t') + '\n');
|
||||
|
||||
console.log(`📦 Updated package.json version: ${currentVersion} → ${newVersion}`);
|
||||
return newVersion;
|
||||
};
|
||||
|
||||
/**
|
||||
* Updates version information in manifest.json file.
|
||||
* @param {string} newVersion - Version to update
|
||||
*/
|
||||
const updateManifestVersion = (newVersion) => {
|
||||
const manifestPath = path.resolve(process.cwd(), 'manifest.json');
|
||||
const manifestData = JSON.parse(readFileSync(manifestPath, 'utf8'));
|
||||
|
||||
const currentVersion = manifestData.version;
|
||||
originalManifestVersion = currentVersion; // Store original version for possible rollback
|
||||
manifestData.version = newVersion;
|
||||
|
||||
writeFileSync(manifestPath, JSON.stringify(manifestData, null, '\t') + '\n');
|
||||
console.log(`📋 Updated manifest.json version: ${currentVersion} → ${newVersion}`);
|
||||
};
|
||||
|
||||
/**
|
||||
* Updates version information in versions.json file.
|
||||
* @param {string} previousVersion - Previous version
|
||||
* @param {string} newVersion - Version to update
|
||||
*/
|
||||
const updateVersionsVersion = (previousVersion, newVersion, minAppVersion) => {
|
||||
const versionsPath = path.resolve(process.cwd(), 'versions.json');
|
||||
const versionsData = JSON.parse(readFileSync(versionsPath, 'utf8'));
|
||||
|
||||
const manifestPath = path.resolve(process.cwd(), 'manifest.json');
|
||||
const manifestData = JSON.parse(readFileSync(manifestPath, 'utf8'));
|
||||
|
||||
const currentVersion = manifestData.version;
|
||||
versionsData[newVersion] = minAppVersion;
|
||||
|
||||
writeFileSync(versionsPath, JSON.stringify(versionsData, null, '\t') + '\n');
|
||||
console.log(`📋 Updated versions.json version: ${previousVersion} → ${newVersion}`);
|
||||
};
|
||||
|
||||
/**
|
||||
* Run project build
|
||||
*/
|
||||
const buildProject = () => {
|
||||
try {
|
||||
console.log('🔨 Starting project build...');
|
||||
execSync('npm run build', { stdio: 'inherit' });
|
||||
console.log('✅ Build completed');
|
||||
return true;
|
||||
} catch (error) {
|
||||
console.error('❌ Build failed:', error.message);
|
||||
return false;
|
||||
}
|
||||
};
|
||||
|
||||
/**
|
||||
* Rollback version changes if the release process fails
|
||||
*/
|
||||
const rollbackVersions = () => {
|
||||
if (originalPackageVersion) {
|
||||
try {
|
||||
const packagePath = path.resolve(process.cwd(), 'package.json');
|
||||
const packageData = JSON.parse(readFileSync(packagePath, 'utf8'));
|
||||
const currentVersion = packageData.version;
|
||||
|
||||
packageData.version = originalPackageVersion;
|
||||
writeFileSync(packagePath, JSON.stringify(packageData, null, '\t') + '\n');
|
||||
console.log(`♻️ Rolled back package.json version: ${currentVersion} → ${originalPackageVersion}`);
|
||||
} catch (error) {
|
||||
console.error('❌ Failed to rollback package.json version:', error.message);
|
||||
}
|
||||
}
|
||||
|
||||
if (originalManifestVersion) {
|
||||
try {
|
||||
const manifestPath = path.resolve(process.cwd(), 'manifest.json');
|
||||
const manifestData = JSON.parse(readFileSync(manifestPath, 'utf8'));
|
||||
const currentVersion = manifestData.version;
|
||||
|
||||
const versionsPath = path.resolve(process.cwd(), 'versions.json');
|
||||
const versionsData = JSON.parse(readFileSync(versionsPath, 'utf8'));
|
||||
|
||||
// remove last version from JSON
|
||||
let keys = Object.keys(versionsData)
|
||||
delete versionsData[keys[keys.length-1]]
|
||||
|
||||
writeFileSync(versionsPath, JSON.stringify(versionsData, null, '\t') + '\n');
|
||||
console.log(`♻️ Rolled back versions.json version: ${currentVersion} → ${originalManifestVersion}`);
|
||||
} catch (error) {
|
||||
console.error('❌ Failed to rollback versions.json version:', error.message);
|
||||
}
|
||||
}
|
||||
|
||||
if (originalManifestVersion) {
|
||||
try {
|
||||
const manifestPath = path.resolve(process.cwd(), 'manifest.json');
|
||||
const manifestData = JSON.parse(readFileSync(manifestPath, 'utf8'));
|
||||
const currentVersion = manifestData.version;
|
||||
|
||||
manifestData.version = originalManifestVersion;
|
||||
writeFileSync(manifestPath, JSON.stringify(manifestData, null, '\t') + '\n');
|
||||
console.log(`♻️ Rolled back manifest.json version: ${currentVersion} → ${originalManifestVersion}`);
|
||||
} catch (error) {
|
||||
console.error('❌ Failed to rollback manifest.json version:', error.message);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
};
|
||||
|
||||
/**
|
||||
* Create Git commit and tag
|
||||
* @param {string} version - New version
|
||||
*/
|
||||
const createGitCommitAndTag = (version) => {
|
||||
try {
|
||||
// Stage changed files
|
||||
try {
|
||||
execSync('git add package.json manifest.json versions.json', { stdio: 'inherit' });
|
||||
} catch (error) {
|
||||
console.error('❌ Failed to stage package.json, versions.json and manifest.json:', error.message);
|
||||
return false;
|
||||
}
|
||||
|
||||
// Try to stage release files
|
||||
try {
|
||||
execSync(`git add ${RELEASE_FILES.join(' ')}`, { stdio: 'inherit' });
|
||||
} catch (error) {
|
||||
console.warn('⚠️ Note: Some release files could not be staged. This may be normal if they are gitignored.');
|
||||
}
|
||||
|
||||
// Create commit
|
||||
const commitMessage = `chore: release ${version}`;
|
||||
execSync(`git commit -m "${commitMessage}"`, { stdio: 'inherit' });
|
||||
console.log(`✅ Commit created: ${commitMessage}`);
|
||||
|
||||
// Create tag
|
||||
const tagName = `${version}`;
|
||||
execSync(`git tag -a ${tagName} -m "Release ${tagName}"`, { stdio: 'inherit' });
|
||||
console.log(`🏷️ Tag created: ${tagName}`);
|
||||
|
||||
// Push changes
|
||||
execSync('git push', { stdio: 'inherit' });
|
||||
execSync('git push --tags', { stdio: 'inherit' });
|
||||
console.log('🚀 Changes pushed to GitHub');
|
||||
console.log('📦 GitHub Actions will create the release with artifact attestations.');
|
||||
return true; // Successfully created commit, tag, and pushed
|
||||
} catch (error) {
|
||||
console.error('❌ Error during Git operations:', error.message);
|
||||
return false; // Failed to complete Git operations
|
||||
}
|
||||
};
|
||||
|
||||
/**
|
||||
* Check if Git working tree is clean
|
||||
* @returns {boolean} Whether the working tree is clean
|
||||
*/
|
||||
const isGitWorkingTreeClean = () => {
|
||||
try {
|
||||
// Check for uncommitted changes
|
||||
const output = execSync('git status --porcelain', { encoding: 'utf-8' });
|
||||
return output.trim() === '';
|
||||
} catch (error) {
|
||||
console.error('❌ Error checking Git status:', error.message);
|
||||
return false;
|
||||
}
|
||||
};
|
||||
|
||||
/**
|
||||
* Main function
|
||||
*/
|
||||
const main = () => {
|
||||
let success = true;
|
||||
let newVersion = '';
|
||||
|
||||
try {
|
||||
// Check for uncommitted changes
|
||||
if (!isGitWorkingTreeClean()) {
|
||||
console.error('❌ Cannot proceed with release: You have uncommitted changes.');
|
||||
console.log('Please commit or stash your changes before running the release script.');
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
// Check command line arguments
|
||||
const args = process.argv.slice(2);
|
||||
const versionType = args[0] || DEFAULT_VERSION_TYPE;
|
||||
|
||||
if (!Object.values(VERSION_TYPES).includes(versionType)) {
|
||||
console.error(`❌ Invalid version type: ${versionType}`);
|
||||
console.log(`Valid options: ${Object.values(VERSION_TYPES).join(', ')}`);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
let previousVersion = getPreviousVersion()
|
||||
|
||||
// Step 1: Update package.json version
|
||||
try {
|
||||
newVersion = updatePackageVersion(versionType);
|
||||
} catch (error) {
|
||||
console.error('❌ Failed to update package.json version:', error.message);
|
||||
success = false;
|
||||
}
|
||||
|
||||
// Step 2: Update manifest.json version
|
||||
if (success) {
|
||||
try {
|
||||
updateManifestVersion(newVersion);
|
||||
} catch (error) {
|
||||
console.error('❌ Failed to update manifest.json version:', error.message);
|
||||
success = false;
|
||||
}
|
||||
}
|
||||
|
||||
// Step 2.5: Update versions.json version
|
||||
if (success) {
|
||||
try {
|
||||
updateVersionsVersion(previousVersion, newVersion, MIN_APP_VERSION);
|
||||
} catch (error) {
|
||||
console.error('❌ Failed to update versions.json version:', error.message);
|
||||
success = false;
|
||||
}
|
||||
}
|
||||
|
||||
// Step 3: Build the project
|
||||
if (success) {
|
||||
if (!buildProject()) {
|
||||
console.error('❌ Build process failed.');
|
||||
success = false;
|
||||
}
|
||||
}
|
||||
|
||||
// Step 4: Git operations
|
||||
if (success) {
|
||||
if (!createGitCommitAndTag(newVersion)) {
|
||||
console.error('❌ Git operations failed.');
|
||||
success = false;
|
||||
}
|
||||
}
|
||||
|
||||
// Check overall success
|
||||
if (success) {
|
||||
console.log(`\n🎉 Release ${newVersion} completed successfully!`);
|
||||
} else {
|
||||
console.error('❌ Release process failed. Rolling back version changes...');
|
||||
rollbackVersions();
|
||||
process.exit(1);
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('❌ Unexpected error during release process:', error.message);
|
||||
rollbackVersions();
|
||||
process.exit(1);
|
||||
}
|
||||
};
|
||||
|
||||
// Run script
|
||||
main();
|
||||
107
src/comments.ts
Normal file
107
src/comments.ts
Normal file
|
|
@ -0,0 +1,107 @@
|
|||
import { requestUrl } from 'obsidian';
|
||||
import type { SubstackClipperSettings } from './types';
|
||||
|
||||
interface Comment {
|
||||
id: number;
|
||||
user_id: number;
|
||||
name: string;
|
||||
handle: string;
|
||||
photo_url: string | null;
|
||||
body: string | null;
|
||||
date: string;
|
||||
status: string;
|
||||
children?: Comment[];
|
||||
}
|
||||
|
||||
interface CommentsResponse {
|
||||
comments: Comment[];
|
||||
}
|
||||
|
||||
export async function fetchComments(
|
||||
domain: string,
|
||||
postId: number,
|
||||
sort: SubstackClipperSettings['commentSort'],
|
||||
): Promise<CommentsResponse> {
|
||||
const url = `${domain}/api/v1/post/${postId}/comments?token=&all_comments=true&sort=${sort}`;
|
||||
const response = await requestUrl({
|
||||
url,
|
||||
headers: {
|
||||
'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/109.0.0.0 Safari/537.36',
|
||||
},
|
||||
});
|
||||
return response.json as CommentsResponse;
|
||||
}
|
||||
|
||||
function formatDate(isoDate: string): string {
|
||||
const d = new Date(isoDate);
|
||||
const months = ['Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec'];
|
||||
const month = months[d.getMonth()];
|
||||
const day = String(d.getDate()).padStart(2, '0');
|
||||
const year = d.getFullYear();
|
||||
let hours = d.getHours();
|
||||
const ampm = hours >= 12 ? 'PM' : 'AM';
|
||||
hours = hours % 12 || 12;
|
||||
const minutes = String(d.getMinutes()).padStart(2, '0');
|
||||
return `${month} ${day}, ${year} ${hours}:${minutes} ${ampm}`;
|
||||
}
|
||||
|
||||
const DEFAULT_AVATAR = '';
|
||||
|
||||
function encodePhotoUrl(photoUrl: string): string {
|
||||
return encodeURIComponent(photoUrl);
|
||||
}
|
||||
|
||||
function renderComment(
|
||||
comment: Comment,
|
||||
slug: string,
|
||||
domain: string,
|
||||
level: number,
|
||||
): string {
|
||||
let md = '';
|
||||
const indent = '> '.repeat(level + 1);
|
||||
const commentUrl = `${domain}/p/${slug}/comment/${comment.id}`;
|
||||
|
||||
if (level === 0) {
|
||||
md += '\n\n---\n\n';
|
||||
}
|
||||
|
||||
const dateStr = formatDate(comment.date);
|
||||
|
||||
if (comment.status === 'deleted' || comment.status === 'flagged' || comment.status === 'moderator_removed') {
|
||||
md += `${indent}${DEFAULT_AVATAR} Comment removed • [${dateStr}](${commentUrl})`;
|
||||
md += `\n${indent}`;
|
||||
md += `\n${indent} Comment removed\n${indent}`;
|
||||
md += `\n${indent}\n\n`;
|
||||
} else {
|
||||
const avatar = comment.photo_url
|
||||
? `})`
|
||||
: DEFAULT_AVATAR;
|
||||
|
||||
md += `${indent}${avatar} [${comment.name}](https://substack.com/profile/${comment.user_id}-${comment.handle}) • [${dateStr}](${commentUrl})`;
|
||||
md += `\n${indent}`;
|
||||
if (comment.body) {
|
||||
md += `\n${indent} ${comment.body.replace(/\n/g, '\n' + indent)}`;
|
||||
}
|
||||
md += '\n\n';
|
||||
}
|
||||
|
||||
if (comment.children) {
|
||||
for (const child of comment.children) {
|
||||
md += renderComment(child, slug, domain, level + 1);
|
||||
}
|
||||
}
|
||||
|
||||
return md;
|
||||
}
|
||||
|
||||
export function renderComments(
|
||||
data: CommentsResponse,
|
||||
slug: string,
|
||||
domain: string,
|
||||
): string {
|
||||
let md = '';
|
||||
for (const comment of data.comments) {
|
||||
md += renderComment(comment, slug, domain, 0);
|
||||
}
|
||||
return md;
|
||||
}
|
||||
95
src/converter.ts
Normal file
95
src/converter.ts
Normal file
|
|
@ -0,0 +1,95 @@
|
|||
import TurndownService from 'turndown';
|
||||
import { imageFilename } from './downloader';
|
||||
|
||||
function isSubstackImageUrl(url: string): boolean {
|
||||
return url.includes('substackcdn.com/image') ||
|
||||
url.includes('.s3.amazonaws.com/public/images');
|
||||
}
|
||||
|
||||
export function createConverter(downloadedUrls: Set<string>): TurndownService {
|
||||
const turndown = new TurndownService({
|
||||
headingStyle: 'atx',
|
||||
hr: '---',
|
||||
bulletListMarker: '-',
|
||||
codeBlockStyle: 'fenced',
|
||||
});
|
||||
|
||||
// Linked Substack images: <a href="cdn"><img src="cdn"></a>
|
||||
turndown.addRule('substackLinkedImage', {
|
||||
filter: (node) => {
|
||||
if (node.nodeName !== 'A') return false;
|
||||
if (!node.querySelector('img')) return false;
|
||||
const href = node.getAttribute('href') ?? '';
|
||||
return isSubstackImageUrl(href);
|
||||
},
|
||||
replacement: (_content, node) => {
|
||||
const img = node.querySelector('img');
|
||||
const src = img?.getAttribute('src') ?? node.getAttribute('href') ?? '';
|
||||
if (downloadedUrls.has(src)) {
|
||||
return `\n\n![[${imageFilename(src)}]]\n\n`;
|
||||
}
|
||||
const alt = img?.getAttribute('alt') ?? '';
|
||||
return `\n\n\n\n`;
|
||||
},
|
||||
});
|
||||
|
||||
// Standalone Substack images: <img src="cdn">
|
||||
turndown.addRule('substackImage', {
|
||||
filter: (node) => {
|
||||
if (node.nodeName !== 'IMG') return false;
|
||||
if (node.parentElement?.nodeName === 'A') return false;
|
||||
const src = node.getAttribute('src') ?? '';
|
||||
return isSubstackImageUrl(src);
|
||||
},
|
||||
replacement: (_content, node) => {
|
||||
const src = node.getAttribute('src') ?? '';
|
||||
if (downloadedUrls.has(src)) {
|
||||
return `\n\n![[${imageFilename(src)}]]\n\n`;
|
||||
}
|
||||
const alt = node.getAttribute('alt') ?? '';
|
||||
return `\n\n\n\n`;
|
||||
},
|
||||
});
|
||||
|
||||
turndown.addRule('iframe', {
|
||||
filter: 'iframe',
|
||||
replacement: (_content, node) => {
|
||||
return '\n\n' + node.outerHTML + '\n\n';
|
||||
},
|
||||
});
|
||||
|
||||
turndown.addRule('audio', {
|
||||
filter: 'audio',
|
||||
replacement: (_content, node) => {
|
||||
return node.outerHTML;
|
||||
},
|
||||
});
|
||||
|
||||
turndown.addRule('videoEmbed', {
|
||||
filter: (node) => {
|
||||
return node.nodeName === 'DIV' &&
|
||||
node.getAttribute('data-component-name') === 'VideoEmbedPlayer';
|
||||
},
|
||||
replacement: (_content, node) => {
|
||||
return node.outerHTML;
|
||||
},
|
||||
});
|
||||
|
||||
turndown.addRule('mediaDiv', {
|
||||
filter: (node) => {
|
||||
if (node.nodeName !== 'DIV') return false;
|
||||
const id = node.getAttribute('id') ?? '';
|
||||
return id.startsWith('media-') && id.length === 42;
|
||||
},
|
||||
replacement: (_content, node) => {
|
||||
return node.outerHTML;
|
||||
},
|
||||
});
|
||||
|
||||
return turndown;
|
||||
}
|
||||
|
||||
export function htmlToMarkdown(html: string, downloadedUrls: Set<string>): string {
|
||||
const converter = createConverter(downloadedUrls);
|
||||
return converter.turndown(html);
|
||||
}
|
||||
196
src/downloader.ts
Normal file
196
src/downloader.ts
Normal file
|
|
@ -0,0 +1,196 @@
|
|||
import { App, normalizePath, requestUrl } from 'obsidian';
|
||||
import type { ParsedArticle, DownloadResult } from './types';
|
||||
|
||||
const USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/109.0.0.0 Safari/537.36';
|
||||
|
||||
async function ensureFolder(app: App, folderPath: string): Promise<void> {
|
||||
const normalized = normalizePath(folderPath);
|
||||
if (!app.vault.getFolderByPath(normalized)) {
|
||||
await app.vault.createFolder(normalized);
|
||||
}
|
||||
}
|
||||
|
||||
async function exceedsMaxSize(url: string, maxSizeMB: number): Promise<boolean> {
|
||||
if (maxSizeMB <= 0) return false;
|
||||
try {
|
||||
const response = await requestUrl({
|
||||
url,
|
||||
method: 'HEAD',
|
||||
headers: { 'User-Agent': USER_AGENT },
|
||||
});
|
||||
const contentLength = response.headers['content-length'] ?? response.headers['Content-Length'];
|
||||
if (contentLength) {
|
||||
return parseInt(contentLength, 10) > maxSizeMB * 1024 * 1024;
|
||||
}
|
||||
} catch {
|
||||
// HEAD failed or unsupported — proceed with download
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
async function downloadToVault(
|
||||
app: App,
|
||||
url: string,
|
||||
vaultPath: string,
|
||||
maxSizeMB: number,
|
||||
): Promise<boolean> {
|
||||
const normalized = normalizePath(vaultPath);
|
||||
if (app.vault.getAbstractFileByPath(normalized)) {
|
||||
return true;
|
||||
}
|
||||
|
||||
if (await exceedsMaxSize(url, maxSizeMB)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
const folderPath = normalized.substring(0, normalized.lastIndexOf('/'));
|
||||
await ensureFolder(app, folderPath);
|
||||
|
||||
const response = await requestUrl({
|
||||
url,
|
||||
headers: { 'User-Agent': USER_AGENT },
|
||||
});
|
||||
|
||||
await app.vault.createBinary(normalized, response.arrayBuffer);
|
||||
return true;
|
||||
}
|
||||
|
||||
function stripS3Prefix(url: string): string {
|
||||
return url
|
||||
.replace(/https%3A%2F%2F[a-z0-9-]+\.s3\.amazonaws\.com%2Fpublic%2Fimages%2F/g, '')
|
||||
.replace(/https:\/\/[a-z0-9-]+\.s3\.amazonaws\.com\/public\/images\//g, '');
|
||||
}
|
||||
|
||||
export function imageFilename(url: string): string {
|
||||
const path = new URL(url).pathname;
|
||||
let basename = path.split('/').pop() ?? 'image.jpg';
|
||||
basename = stripS3Prefix(decodeURIComponent(basename));
|
||||
if (!basename.includes('.')) {
|
||||
basename += '.jpg';
|
||||
}
|
||||
return basename;
|
||||
}
|
||||
|
||||
function videoFilename(url: string): string {
|
||||
const parts = url.split('/');
|
||||
return `${parts[7]}.mp4`;
|
||||
}
|
||||
|
||||
function audioFilename(url: string): string {
|
||||
const parts = url.split('/');
|
||||
return `${parts[7]}.mp3`;
|
||||
}
|
||||
|
||||
function podcastAudioFilename(podcastUrl: string): string {
|
||||
if (podcastUrl.includes('post_id')) {
|
||||
const parts = podcastUrl.split('/');
|
||||
const id = parts[5].split('.')[0];
|
||||
return `${id}.mp3`;
|
||||
}
|
||||
const parts = podcastUrl.split('/');
|
||||
return `${parts[7]}.mp3`;
|
||||
}
|
||||
|
||||
function podcastVideoFilename(videoUploadId: string): string {
|
||||
return `${videoUploadId.replace(/"/g, '')}.mp4`;
|
||||
}
|
||||
|
||||
function transcriptFilename(article: ParsedArticle): string {
|
||||
if (article.videoUploadId) {
|
||||
const base = article.transcript.split('?')[0];
|
||||
const basename = base.split('/').pop() ?? 'transcript.vtt';
|
||||
return `${article.videoUploadId}.${basename}`;
|
||||
}
|
||||
if (article.podcastUrl) {
|
||||
const parts = article.podcastUrl.split('/');
|
||||
return `${parts[7]}.en.vtt`;
|
||||
}
|
||||
return 'transcript.vtt';
|
||||
}
|
||||
|
||||
function attachmentFilename(url: string): string {
|
||||
const parsed = new URL(url);
|
||||
return parsed.pathname.split('/').pop() ?? 'attachment';
|
||||
}
|
||||
|
||||
export async function downloadAllMedia(
|
||||
app: App,
|
||||
article: ParsedArticle,
|
||||
saveDir: string,
|
||||
downloadMedia: boolean,
|
||||
maxFileSize: number,
|
||||
): Promise<DownloadResult> {
|
||||
const downloaded = new Set<string>();
|
||||
const skipped = new Set<string>();
|
||||
|
||||
if (!downloadMedia) {
|
||||
for (const url of article.images) skipped.add(url);
|
||||
if (article.coverImage) skipped.add(article.coverImage);
|
||||
for (const url of article.attachments) skipped.add(url);
|
||||
for (const url of article.videos) skipped.add(url);
|
||||
for (const url of article.audios) skipped.add(url);
|
||||
if (article.podcastUrl) skipped.add(article.podcastUrl);
|
||||
if (article.videoUploadId) {
|
||||
skipped.add(`https://api.substack.com/api/v1/video/upload/${article.videoUploadId}/src`);
|
||||
}
|
||||
if (article.transcript) skipped.add(article.transcript);
|
||||
return { downloaded, skipped };
|
||||
}
|
||||
|
||||
const tasks: Promise<{ url: string; ok: boolean }>[] = [];
|
||||
|
||||
function track(url: string, vaultPath: string): void {
|
||||
tasks.push(
|
||||
downloadToVault(app, url, vaultPath, maxFileSize)
|
||||
.then(ok => ({ url, ok }))
|
||||
.catch(() => ({ url, ok: false })),
|
||||
);
|
||||
}
|
||||
|
||||
for (const url of article.images) {
|
||||
if (url.includes('/img/missing-image.png')) continue;
|
||||
track(url, `${saveDir}/${imageFilename(url)}`);
|
||||
}
|
||||
|
||||
if (article.coverImage) {
|
||||
track(article.coverImage, `${saveDir}/${imageFilename(article.coverImage)}`);
|
||||
}
|
||||
|
||||
for (const url of article.attachments) {
|
||||
track(url, `${saveDir}/${attachmentFilename(url)}`);
|
||||
}
|
||||
|
||||
for (const url of article.videos) {
|
||||
track(url, `${saveDir}/${videoFilename(url)}`);
|
||||
}
|
||||
|
||||
for (const url of article.audios) {
|
||||
track(url, `${saveDir}/${audioFilename(url)}`);
|
||||
}
|
||||
|
||||
if (article.podcastUrl) {
|
||||
track(article.podcastUrl, `${saveDir}/${podcastAudioFilename(article.podcastUrl)}`);
|
||||
}
|
||||
|
||||
if (article.videoUploadId) {
|
||||
const videoUrl = `https://api.substack.com/api/v1/video/upload/${article.videoUploadId}/src`;
|
||||
track(videoUrl, `${saveDir}/${podcastVideoFilename(article.videoUploadId)}`);
|
||||
}
|
||||
|
||||
if (article.transcript) {
|
||||
track(article.transcript, `${saveDir}/${transcriptFilename(article)}`);
|
||||
}
|
||||
|
||||
const results = await Promise.allSettled(tasks);
|
||||
for (const result of results) {
|
||||
if (result.status === 'fulfilled') {
|
||||
if (result.value.ok) {
|
||||
downloaded.add(result.value.url);
|
||||
} else {
|
||||
skipped.add(result.value.url);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return { downloaded, skipped };
|
||||
}
|
||||
268
src/main.ts
Normal file
268
src/main.ts
Normal file
|
|
@ -0,0 +1,268 @@
|
|||
import { normalizePath, Notice, Plugin } from 'obsidian';
|
||||
import { type SubstackClipperSettings, DEFAULT_SETTINGS } from './types';
|
||||
import type { ParsedArticle, DownloadResult } from './types';
|
||||
import { SettingsTab } from './settings';
|
||||
import { ClipModal } from './modal';
|
||||
import { fetchHtml, extractPreloads, parsePost, parseUrl, extractImages, extractVideos, extractAudios, extractAttachments } from './parser';
|
||||
import { htmlToMarkdown } from './converter';
|
||||
import { postprocessMarkdown } from './postprocess';
|
||||
import { downloadAllMedia } from './downloader';
|
||||
import { fetchComments, renderComments } from './comments';
|
||||
|
||||
export default class SubstackClipperPlugin extends Plugin {
|
||||
settings: SubstackClipperSettings;
|
||||
|
||||
async onload(): Promise<void> {
|
||||
await this.loadSettings();
|
||||
this.addSettingTab(new SettingsTab(this.app, this));
|
||||
|
||||
this.addCommand({
|
||||
id: 'clip-post',
|
||||
name: 'Clip substack post',
|
||||
callback: () => {
|
||||
new ClipModal(this.app, (url) => {
|
||||
void this.clipPost(url);
|
||||
}).open();
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
async loadSettings(): Promise<void> {
|
||||
this.settings = Object.assign({}, DEFAULT_SETTINGS, await this.loadData() as Partial<SubstackClipperSettings>);
|
||||
}
|
||||
|
||||
async saveSettings(): Promise<void> {
|
||||
await this.saveData(this.settings);
|
||||
}
|
||||
|
||||
private async clipPost(url: string): Promise<void> {
|
||||
const notice = new Notice('Clipping...', 0);
|
||||
|
||||
try {
|
||||
const { username, slug, domain } = parseUrl(url);
|
||||
const html = await fetchHtml(url);
|
||||
const preloads = extractPreloads(html);
|
||||
const post = parsePost(html, preloads);
|
||||
|
||||
if (!post.contentHtml) {
|
||||
new Notice('Could not find article content.');
|
||||
}
|
||||
|
||||
const images = extractImages(post.contentHtml);
|
||||
const videos = extractVideos(post.contentHtml);
|
||||
const audios = extractAudios(post.contentHtml);
|
||||
const attachments = extractAttachments(post.contentHtml);
|
||||
|
||||
const article: ParsedArticle = {
|
||||
...post,
|
||||
html: post.contentHtml,
|
||||
markdown: '',
|
||||
images,
|
||||
videos,
|
||||
audios,
|
||||
attachments,
|
||||
};
|
||||
|
||||
const saveDir = normalizePath(`${this.settings.saveDirectory}/${username}/${slug}`);
|
||||
|
||||
notice.setMessage('Downloading media...');
|
||||
const downloadResult: DownloadResult = await downloadAllMedia(
|
||||
this.app, article, saveDir,
|
||||
this.settings.downloadMedia, this.settings.maxFileSize,
|
||||
);
|
||||
|
||||
const markdown = htmlToMarkdown(post.contentHtml, downloadResult.downloaded);
|
||||
article.markdown = markdown;
|
||||
|
||||
// Comments
|
||||
let discussionEmbed = '';
|
||||
if (this.settings.downloadComments) {
|
||||
notice.setMessage('Fetching comments...');
|
||||
try {
|
||||
const commentsData = await fetchComments(domain, article.id, this.settings.commentSort);
|
||||
|
||||
if (this.settings.saveRawJson) {
|
||||
const commentsJsonPath = normalizePath(`${saveDir}/${slug}-comments.json`);
|
||||
await this.writeFile(commentsJsonPath, JSON.stringify(commentsData, null, 2));
|
||||
}
|
||||
|
||||
if (commentsData.comments && commentsData.comments.length > 0) {
|
||||
const commentsMd = renderComments(commentsData, slug, domain);
|
||||
const commentsPath = normalizePath(`${this.settings.saveDirectory}/${username}/${slug}-comments.md`);
|
||||
await this.writeFile(commentsPath, commentsMd);
|
||||
discussionEmbed = `\n\n## Discussion\n\n![[${slug}-comments]]`;
|
||||
}
|
||||
} catch (e) {
|
||||
console.error('[substack-clipper] comments error:', e);
|
||||
new Notice(`Failed to fetch comments: ${e instanceof Error ? e.message : String(e)}`);
|
||||
}
|
||||
}
|
||||
|
||||
let processedMd = postprocessMarkdown(markdown, domain, downloadResult.downloaded);
|
||||
|
||||
// Build podcast/video/transcript sections
|
||||
let podcastVideoPlayer = '';
|
||||
let podcastAudioPlayer = '';
|
||||
let transcriptLink = '';
|
||||
|
||||
if (article.videoUploadId) {
|
||||
const cleanId = article.videoUploadId.replace(/"/g, '');
|
||||
const videoUrl = `https://api.substack.com/api/v1/video/upload/${cleanId}/src`;
|
||||
if (downloadResult.downloaded.has(videoUrl)) {
|
||||
podcastVideoPlayer = `## Podcast video\n\n![[${cleanId}.mp4]]\n\n`;
|
||||
} else {
|
||||
podcastVideoPlayer = `## Podcast video\n\n[Podcast video](${videoUrl})\n\n`;
|
||||
}
|
||||
}
|
||||
|
||||
if (article.podcastUrl) {
|
||||
if (downloadResult.downloaded.has(article.podcastUrl)) {
|
||||
const parts = article.podcastUrl.split('/');
|
||||
if (article.podcastUrl.includes('post_id')) {
|
||||
const id = parts[5].split('.')[0];
|
||||
podcastAudioPlayer = `## Podcast audio\n\n![[${id}.mp3]]\n\n`;
|
||||
} else {
|
||||
podcastAudioPlayer = `## Podcast audio\n\n![[${parts[7]}.mp3]]\n\n`;
|
||||
}
|
||||
} else {
|
||||
podcastAudioPlayer = `## Podcast audio\n\n[Podcast audio](${article.podcastUrl})\n\n`;
|
||||
}
|
||||
}
|
||||
|
||||
if (article.transcript) {
|
||||
if (downloadResult.downloaded.has(article.transcript)) {
|
||||
if (article.videoUploadId) {
|
||||
const base = article.transcript.split('?')[0];
|
||||
const basename = base.split('/').pop() ?? 'transcript.vtt';
|
||||
transcriptLink = `### Transcript\n\n[[${article.videoUploadId}.${basename}]]\n\n`;
|
||||
} else if (article.podcastUrl) {
|
||||
const parts = article.podcastUrl.split('/');
|
||||
transcriptLink = `### Transcript\n\n[[${parts[7]}.en.vtt]]\n\n`;
|
||||
}
|
||||
} else {
|
||||
transcriptLink = `### Transcript\n\n[Transcript](${article.transcript})\n\n`;
|
||||
}
|
||||
}
|
||||
|
||||
// Build frontmatter
|
||||
const frontmatter = this.buildFrontmatter(article, url);
|
||||
|
||||
const finalNote = [
|
||||
'---',
|
||||
frontmatter,
|
||||
'---\n',
|
||||
podcastVideoPlayer,
|
||||
podcastAudioPlayer,
|
||||
transcriptLink,
|
||||
processedMd,
|
||||
discussionEmbed,
|
||||
].filter(Boolean).join('\n');
|
||||
|
||||
const notePath = normalizePath(`${this.settings.saveDirectory}/${username}/${slug}.md`);
|
||||
await this.writeFile(notePath, finalNote);
|
||||
|
||||
// Optional raw files
|
||||
if (this.settings.saveRawJson) {
|
||||
const jsonPath = normalizePath(`${saveDir}/${slug}.json`);
|
||||
await this.writeFile(jsonPath, JSON.stringify({
|
||||
title: article.title,
|
||||
subtitle: article.subtitle,
|
||||
type: article.type,
|
||||
audience: article.audience,
|
||||
link: url,
|
||||
date: article.date,
|
||||
id: article.id,
|
||||
cover_image: article.coverImage,
|
||||
videos: article.videos,
|
||||
audios: article.audios,
|
||||
attachments: article.attachments,
|
||||
podcast: article.podcastUrl,
|
||||
video_upload_id: article.videoUploadId,
|
||||
transcript: article.transcript,
|
||||
comment_count: article.commentCount,
|
||||
md: processedMd,
|
||||
}, null, 2));
|
||||
}
|
||||
|
||||
if (this.settings.saveRawHtml) {
|
||||
const htmlPath = normalizePath(`${saveDir}/${slug}.html`);
|
||||
await this.writeFile(htmlPath, article.html);
|
||||
}
|
||||
|
||||
notice.hide();
|
||||
new Notice(`Clipped: ${article.title}`);
|
||||
|
||||
} catch (e) {
|
||||
notice.hide();
|
||||
new Notice(`Clip failed: ${e instanceof Error ? e.message : String(e)}`);
|
||||
}
|
||||
}
|
||||
|
||||
private buildFrontmatter(article: ParsedArticle, url: string): string {
|
||||
const lines: string[] = [];
|
||||
|
||||
lines.push(`title: ${JSON.stringify(article.title.trim())}`);
|
||||
|
||||
if (article.subtitle) {
|
||||
lines.push(`subtitle: ${JSON.stringify(article.subtitle.trim())}`);
|
||||
}
|
||||
|
||||
lines.push(`type: ${JSON.stringify(article.type)}`);
|
||||
lines.push(`audience: ${JSON.stringify(article.audience)}`);
|
||||
lines.push(`link: ${JSON.stringify(url)}`);
|
||||
lines.push(`id: ${article.id}`);
|
||||
lines.push(`comment-count: ${article.commentCount}`);
|
||||
|
||||
if (article.videos.length) {
|
||||
lines.push('videos:');
|
||||
for (const v of article.videos) lines.push(` - ${v}`);
|
||||
}
|
||||
|
||||
if (article.audios.length) {
|
||||
lines.push('audios:');
|
||||
for (const a of article.audios) lines.push(` - ${a}`);
|
||||
}
|
||||
|
||||
if (article.attachments.length) {
|
||||
lines.push('attachments:');
|
||||
for (const a of article.attachments) lines.push(` - ${a}`);
|
||||
}
|
||||
|
||||
if (article.podcastUrl) {
|
||||
lines.push(`podcast-audio: ${JSON.stringify(article.podcastUrl)}`);
|
||||
}
|
||||
|
||||
if (article.videoUploadId) {
|
||||
const cleanId = article.videoUploadId.replace(/"/g, '');
|
||||
lines.push(`podcast-video: "https://api.substack.com/api/v1/video/upload/${cleanId}/src"`);
|
||||
}
|
||||
|
||||
if (article.transcript) {
|
||||
lines.push(`transcript: ${JSON.stringify(article.transcript)}`);
|
||||
}
|
||||
|
||||
lines.push(`date: ${JSON.stringify(article.date)}`);
|
||||
|
||||
return lines.join('\n');
|
||||
}
|
||||
|
||||
private async ensureFolder(folderPath: string): Promise<void> {
|
||||
const normalized = normalizePath(folderPath);
|
||||
if (!this.app.vault.getFolderByPath(normalized)) {
|
||||
await this.app.vault.createFolder(normalized);
|
||||
}
|
||||
}
|
||||
|
||||
private async writeFile(path: string, content: string): Promise<void> {
|
||||
const normalized = normalizePath(path);
|
||||
const folder = normalized.substring(0, normalized.lastIndexOf('/'));
|
||||
await this.ensureFolder(folder);
|
||||
|
||||
const existing = this.app.vault.getAbstractFileByPath(normalized);
|
||||
if (existing) {
|
||||
await this.app.vault.adapter.write(normalized, content);
|
||||
} else {
|
||||
await this.app.vault.create(normalized, content);
|
||||
}
|
||||
}
|
||||
}
|
||||
51
src/modal.ts
Normal file
51
src/modal.ts
Normal file
|
|
@ -0,0 +1,51 @@
|
|||
import { App, Modal, Notice, Setting } from 'obsidian';
|
||||
|
||||
export class ClipModal extends Modal {
|
||||
private url = '';
|
||||
private onSubmit: (url: string) => void;
|
||||
|
||||
constructor(app: App, onSubmit: (url: string) => void) {
|
||||
super(app);
|
||||
this.onSubmit = onSubmit;
|
||||
}
|
||||
|
||||
onOpen(): void {
|
||||
const { contentEl } = this;
|
||||
contentEl.addClass('substack-clipper-modal');
|
||||
contentEl.createEl('h2', { text: 'Clip substack post' });
|
||||
|
||||
new Setting(contentEl)
|
||||
.setName('Post URL')
|
||||
.setDesc('Full URL to a substack post (must contain /p/).')
|
||||
.addText(text => {
|
||||
text.setPlaceholder('Enter substack post URL');
|
||||
text.onChange(value => { this.url = value.trim(); });
|
||||
text.inputEl.addEventListener('keydown', (e: KeyboardEvent) => {
|
||||
if (e.key === 'Enter') {
|
||||
e.preventDefault();
|
||||
this.submit();
|
||||
}
|
||||
});
|
||||
text.inputEl.setCssStyles({ width: '100%' });
|
||||
});
|
||||
|
||||
new Setting(contentEl)
|
||||
.addButton(btn => btn
|
||||
.setButtonText('Clip')
|
||||
.setCta()
|
||||
.onClick(() => { this.submit(); }));
|
||||
}
|
||||
|
||||
private submit(): void {
|
||||
if (!this.url || !this.url.includes('/p/')) {
|
||||
new Notice('Invalid URL — must contain /p/ path segment.');
|
||||
return;
|
||||
}
|
||||
this.close();
|
||||
this.onSubmit(this.url);
|
||||
}
|
||||
|
||||
onClose(): void {
|
||||
this.contentEl.empty();
|
||||
}
|
||||
}
|
||||
138
src/parser.ts
Normal file
138
src/parser.ts
Normal file
|
|
@ -0,0 +1,138 @@
|
|||
import { requestUrl } from 'obsidian';
|
||||
import type { SubstackPost } from './types';
|
||||
|
||||
interface PreloadsJson {
|
||||
post: Record<string, unknown>;
|
||||
}
|
||||
|
||||
export function parseUrl(url: string): { username: string; slug: string; domain: string } {
|
||||
const parsed = new URL(url);
|
||||
const slug = parsed.pathname.split('/p/')[1]?.replace(/\/$/, '') ?? '';
|
||||
const username = parsed.hostname.endsWith('.substack.com')
|
||||
? parsed.hostname.split('.')[0]
|
||||
: parsed.hostname;
|
||||
const domain = `${parsed.protocol}//${parsed.hostname}`;
|
||||
return { username, slug, domain };
|
||||
}
|
||||
|
||||
export async function fetchHtml(url: string): Promise<string> {
|
||||
const response = await requestUrl({ url });
|
||||
return response.text;
|
||||
}
|
||||
|
||||
export function extractPreloads(html: string): PreloadsJson {
|
||||
const parser = new DOMParser();
|
||||
const doc = parser.parseFromString(html, 'text/html');
|
||||
const scripts = doc.querySelectorAll('script');
|
||||
|
||||
let jsonString = '';
|
||||
for (const script of Array.from(scripts)) {
|
||||
const text = script.textContent ?? '';
|
||||
if (text.startsWith('window._preloads')) {
|
||||
jsonString = text.substring(38);
|
||||
jsonString = jsonString.slice(0, -2);
|
||||
jsonString = jsonString.replace(/\\"/g, '"');
|
||||
jsonString = jsonString.replace(/\\\\"/g, '\\"');
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (!jsonString) {
|
||||
throw new Error('Could not find window._preloads in page.');
|
||||
}
|
||||
|
||||
return JSON.parse(jsonString) as PreloadsJson;
|
||||
}
|
||||
|
||||
export function parsePost(html: string, json: PreloadsJson): SubstackPost & { contentHtml: string } {
|
||||
const post = json.post;
|
||||
|
||||
const parser = new DOMParser();
|
||||
const doc = parser.parseFromString(html, 'text/html');
|
||||
const content = doc.querySelector('div.markup');
|
||||
|
||||
let signedCaptions = '';
|
||||
const podcastUpload = post['podcastUpload'] as Record<string, unknown> | null | undefined;
|
||||
if (podcastUpload) {
|
||||
const transcription = podcastUpload['transcription'] as Record<string, unknown> | null | undefined;
|
||||
if (transcription) {
|
||||
const captions = transcription['signed_captions'] as Array<{ url: string }> | undefined;
|
||||
if (captions?.[0]) {
|
||||
signedCaptions = captions[0].url;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
title: (post['title'] as string) ?? '',
|
||||
subtitle: (post['subtitle'] as string) ?? '',
|
||||
type: (post['type'] as string) ?? '',
|
||||
audience: (post['audience'] as string) ?? '',
|
||||
url: (post['canonical_url'] as string) ?? '',
|
||||
slug: (post['slug'] as string) ?? '',
|
||||
coverImage: (post['cover_image'] as string) ?? '',
|
||||
date: (post['post_date'] as string) ?? '',
|
||||
id: (post['id'] as number) ?? 0,
|
||||
wordcount: (post['wordcount'] as number) ?? 0,
|
||||
commentCount: (post['comment_count'] as number) ?? 0,
|
||||
podcastUrl: (post['podcast_url'] as string) ?? null,
|
||||
videoUploadId: (post['video_upload_id'] as string) ?? null,
|
||||
transcript: signedCaptions,
|
||||
contentHtml: content ? content.innerHTML : '',
|
||||
};
|
||||
}
|
||||
|
||||
export function extractImages(contentHtml: string): string[] {
|
||||
const parser = new DOMParser();
|
||||
const doc = parser.parseFromString(contentHtml, 'text/html');
|
||||
const imgs: string[] = [];
|
||||
for (const img of Array.from(doc.querySelectorAll('img'))) {
|
||||
const src = img.getAttribute('src') ?? '';
|
||||
if (src && !src.includes('attachment_icon.svg')) {
|
||||
imgs.push(src);
|
||||
}
|
||||
}
|
||||
return imgs;
|
||||
}
|
||||
|
||||
export function extractVideos(contentHtml: string): string[] {
|
||||
const parser = new DOMParser();
|
||||
const doc = parser.parseFromString(contentHtml, 'text/html');
|
||||
const videos: string[] = [];
|
||||
for (const div of Array.from(doc.querySelectorAll('div[id^="media-"]'))) {
|
||||
const id = div.getAttribute('id') ?? '';
|
||||
if (id.length === 42) {
|
||||
const videoId = id.replace('media-', '');
|
||||
videos.push(`https://substack.com/api/v1/video/upload/${videoId}/src`);
|
||||
}
|
||||
}
|
||||
return videos;
|
||||
}
|
||||
|
||||
export function extractAudios(contentHtml: string): string[] {
|
||||
const parser = new DOMParser();
|
||||
const doc = parser.parseFromString(contentHtml, 'text/html');
|
||||
const audios: string[] = [];
|
||||
for (const audio of Array.from(doc.querySelectorAll('audio'))) {
|
||||
const src = audio.getAttribute('src') ?? '';
|
||||
if (src) {
|
||||
audios.push(`https://substack.com${src}`);
|
||||
}
|
||||
}
|
||||
return [...new Set(audios)];
|
||||
}
|
||||
|
||||
export function extractAttachments(contentHtml: string): string[] {
|
||||
const parser = new DOMParser();
|
||||
const doc = parser.parseFromString(contentHtml, 'text/html');
|
||||
const attachments: string[] = [];
|
||||
for (const a of Array.from(doc.querySelectorAll('a.file-embed-button.wide'))) {
|
||||
const href = a.getAttribute('href') ?? '';
|
||||
if (href) attachments.push(href);
|
||||
}
|
||||
for (const a of Array.from(doc.querySelectorAll('a[href$=".pdf"]'))) {
|
||||
const href = a.getAttribute('href') ?? '';
|
||||
if (href) attachments.push(href);
|
||||
}
|
||||
return [...new Set(attachments)];
|
||||
}
|
||||
98
src/postprocess.ts
Normal file
98
src/postprocess.ts
Normal file
|
|
@ -0,0 +1,98 @@
|
|||
export function postprocessMarkdown(md: string, domain: string, downloadedUrls: Set<string>): string {
|
||||
let out = md;
|
||||
|
||||
// iframe spacing
|
||||
out = out.replace(/<iframe/g, '\n\n<iframe');
|
||||
out = out.replace(/<\/iframe>/g, '</iframe>\n\n');
|
||||
|
||||
// fix missing line breaks after image/link closings
|
||||
out = out.replace(/(\.[a-z]{3,4}\))([a-zA-Z0-9!#""[*…<])/g, '$1\n\n$2');
|
||||
|
||||
// fix headings glued to links
|
||||
out = out.replace(/(\))(#{1,6})/g, '$1\n\n$2');
|
||||
|
||||
// footnotes
|
||||
const escapedDomain = domain.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
|
||||
out = out.replace(
|
||||
new RegExp(`\\[(\\d)\\]\\(${escapedDomain}/p/([a-zA-Z0-9-]+)#footnote-\\d-\\d+\\)`, 'g'),
|
||||
'[^$1]',
|
||||
);
|
||||
out = out.replace(
|
||||
new RegExp(`\\[(\\d)\\]\\(${escapedDomain}/p/([a-zA-Z0-9-]+)#footnote-anchor-\\d-\\d+\\)`, 'g'),
|
||||
'[^$1]: ',
|
||||
);
|
||||
|
||||
// truncated post previews
|
||||
out = out.replace(
|
||||
/\[(.*)\n(\n)?-+\]\([a-z0-9-:/.]+\)(.*)(\n\n)?\[Read full story\]\(([a-z0-9-]+)\)/g,
|
||||
'\n\n[$1]($5)\n\n',
|
||||
);
|
||||
out = out.replace(
|
||||
/\[!\[\]\(.*\)\n\n(.*)…Read more(.*)\]\((.*)\)/g,
|
||||
'\n\n[$1]($3)\n\n',
|
||||
);
|
||||
|
||||
// image cleanup
|
||||
out = out.replace(/!\[None\]/g, '![]');
|
||||
out = out.replace(/ "None"\)/g, ')');
|
||||
out = out.replace(/\)\[!\[\]/g, ')\n\n[![]');
|
||||
|
||||
// CDN/S3 prefix cleanup only when images were downloaded (wikilinks don't contain URLs)
|
||||
if (downloadedUrls.size > 0) {
|
||||
out = out.replace(/https:\/\/substackcdn\.com\/image\/fetch\/[^/]+\//g, '');
|
||||
|
||||
out = out.replace(
|
||||
/\[!\[(.*)?\]\((https%3A%2F%2F[a-z0-9-]+\.s3\.amazonaws\.com%2Fpublic%2Fimages%2F(.*)\.[a-z]{3,4})( ".*")?\)\]\((https%3A%2F%2F[a-z0-9-]+\.s3\.amazonaws\.com%2Fpublic%2Fimages%2F(.*)\.[a-z]{3,4})\)/g,
|
||||
'',
|
||||
);
|
||||
|
||||
out = out.replace(
|
||||
/(https%3A%2F%2F[a-z0-9-]+\.s3\.amazonaws\.com%2Fpublic%2Fimages%2F)/g,
|
||||
'',
|
||||
);
|
||||
}
|
||||
|
||||
// embedded video divs → wikilinks or links
|
||||
out = out.replace(
|
||||
/<div [^>]*id="media-([^"]+)"[^>]*>.*<\/div>/g,
|
||||
(_match, videoId: string) => {
|
||||
const videoUrl = `https://api.substack.com/api/v1/video/upload/${videoId}/src`;
|
||||
if (downloadedUrls.has(videoUrl)) {
|
||||
return `![[${videoId}.mp4]]\n\n`;
|
||||
}
|
||||
return `[Video](${videoUrl})\n\n`;
|
||||
},
|
||||
);
|
||||
|
||||
// embedded audio → wikilinks or links
|
||||
out = out.replace(
|
||||
/[0-9×:-]+<audio [^>]*src="\/api\/v1\/audio\/upload\/([^"]+)\/src"[^>]*>.*<\/audio>/g,
|
||||
(_match, audioId: string) => {
|
||||
const audioUrl = `https://api.substack.com/api/v1/audio/upload/${audioId}/src`;
|
||||
if (downloadedUrls.has(audioUrl)) {
|
||||
return `\n\n![[${audioId}.mp3]]\n\n`;
|
||||
}
|
||||
return `\n\n[Audio](${audioUrl})\n\n`;
|
||||
},
|
||||
);
|
||||
|
||||
// remove preserved spacing notice
|
||||
out = out.replace(/Text within this block will maintain its original spacing when published/g, '');
|
||||
|
||||
// remove attachment icon images
|
||||
out = out.replace(/!\[\]\(https%3A%2F%2Fsubstack\.com%2Fimg%2Fattachment_icon\.svg\) ?\n?/g, '');
|
||||
|
||||
// PDF download links → wikilinks or keep original links
|
||||
out = out.replace(
|
||||
/\[Download\]\(https:\/\/([a-z0-9.]+)\/api\/v1\/file\/([a-z0-9-]+)\.pdf\)/g,
|
||||
(_match, host: string, fileId: string) => {
|
||||
const pdfUrl = `https://${host}/api/v1/file/${fileId}.pdf`;
|
||||
if (downloadedUrls.has(pdfUrl)) {
|
||||
return `[[${fileId}.pdf]]`;
|
||||
}
|
||||
return _match;
|
||||
},
|
||||
);
|
||||
|
||||
return out;
|
||||
}
|
||||
99
src/settings.ts
Normal file
99
src/settings.ts
Normal file
|
|
@ -0,0 +1,99 @@
|
|||
import { App, PluginSettingTab } from 'obsidian';
|
||||
import type { SettingDefinitionItem } from 'obsidian';
|
||||
import type SubstackClipperPlugin from './main';
|
||||
|
||||
export class SettingsTab extends PluginSettingTab {
|
||||
plugin: SubstackClipperPlugin;
|
||||
|
||||
constructor(app: App, plugin: SubstackClipperPlugin) {
|
||||
super(app, plugin);
|
||||
this.plugin = plugin;
|
||||
}
|
||||
|
||||
getSettingDefinitions(): SettingDefinitionItem[] {
|
||||
return [
|
||||
{
|
||||
type: 'group',
|
||||
heading: 'Storage',
|
||||
items: [
|
||||
{
|
||||
name: 'Save directory',
|
||||
desc: 'Vault-relative folder path for clipped posts.',
|
||||
control: {
|
||||
type: 'text',
|
||||
key: 'saveDirectory',
|
||||
placeholder: 'Substacks',
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
type: 'group',
|
||||
heading: 'Downloads',
|
||||
items: [
|
||||
{
|
||||
name: 'Download media',
|
||||
desc: 'Download all media files (images, attachments, videos, audio) to the vault. When off, original Substack URLs are kept.',
|
||||
control: {
|
||||
type: 'toggle',
|
||||
key: 'downloadMedia',
|
||||
},
|
||||
},
|
||||
{
|
||||
name: 'Max file size (MB)',
|
||||
desc: 'Skip media files larger than this size and keep the original URL. 0 means no limit.',
|
||||
control: {
|
||||
type: 'text',
|
||||
key: 'maxFileSize',
|
||||
placeholder: '0',
|
||||
},
|
||||
visible: () => this.plugin.settings.downloadMedia,
|
||||
},
|
||||
{
|
||||
name: 'Download comments',
|
||||
desc: 'Fetch and save threaded comments as a separate note.',
|
||||
control: {
|
||||
type: 'toggle',
|
||||
key: 'downloadComments',
|
||||
},
|
||||
},
|
||||
{
|
||||
name: 'Comment sort order',
|
||||
control: {
|
||||
type: 'dropdown',
|
||||
key: 'commentSort',
|
||||
defaultValue: 'most_recent_first',
|
||||
options: {
|
||||
most_recent_first: 'Most recent first',
|
||||
oldest_first: 'Oldest first',
|
||||
best_first: 'Best first',
|
||||
},
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
type: 'group',
|
||||
heading: 'Advanced',
|
||||
items: [
|
||||
{
|
||||
name: 'Save raw JSON',
|
||||
desc: 'Save the raw Substack API JSON alongside the clipped post.',
|
||||
control: {
|
||||
type: 'toggle',
|
||||
key: 'saveRawJson',
|
||||
},
|
||||
},
|
||||
{
|
||||
name: 'Save raw HTML',
|
||||
desc: 'Save the raw article HTML alongside the clipped post.',
|
||||
control: {
|
||||
type: 'toggle',
|
||||
key: 'saveRawHtml',
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
];
|
||||
}
|
||||
}
|
||||
50
src/types.ts
Normal file
50
src/types.ts
Normal file
|
|
@ -0,0 +1,50 @@
|
|||
export interface SubstackPost {
|
||||
title: string;
|
||||
subtitle: string;
|
||||
type: string;
|
||||
audience: string;
|
||||
url: string;
|
||||
slug: string;
|
||||
coverImage: string;
|
||||
date: string;
|
||||
id: number;
|
||||
wordcount: number;
|
||||
commentCount: number;
|
||||
podcastUrl: string | null;
|
||||
videoUploadId: string | null;
|
||||
transcript: string;
|
||||
}
|
||||
|
||||
export interface ParsedArticle extends SubstackPost {
|
||||
html: string;
|
||||
markdown: string;
|
||||
images: string[];
|
||||
videos: string[];
|
||||
audios: string[];
|
||||
attachments: string[];
|
||||
}
|
||||
|
||||
export interface SubstackClipperSettings {
|
||||
saveDirectory: string;
|
||||
downloadMedia: boolean;
|
||||
maxFileSize: number;
|
||||
downloadComments: boolean;
|
||||
saveRawJson: boolean;
|
||||
saveRawHtml: boolean;
|
||||
commentSort: 'most_recent_first' | 'oldest_first' | 'best_first';
|
||||
}
|
||||
|
||||
export const DEFAULT_SETTINGS: SubstackClipperSettings = {
|
||||
saveDirectory: 'Substacks',
|
||||
downloadMedia: false,
|
||||
maxFileSize: 0,
|
||||
downloadComments: false,
|
||||
saveRawJson: false,
|
||||
saveRawHtml: false,
|
||||
commentSort: 'most_recent_first',
|
||||
};
|
||||
|
||||
export interface DownloadResult {
|
||||
downloaded: Set<string>;
|
||||
skipped: Set<string>;
|
||||
}
|
||||
8
styles.css
Normal file
8
styles.css
Normal file
|
|
@ -0,0 +1,8 @@
|
|||
.substack-clipper-modal .setting-item {
|
||||
border: none;
|
||||
padding: var(--size-4-2) 0;
|
||||
}
|
||||
|
||||
.substack-clipper-modal input[type="text"] {
|
||||
width: 100%;
|
||||
}
|
||||
23
tsconfig.json
Normal file
23
tsconfig.json
Normal file
|
|
@ -0,0 +1,23 @@
|
|||
{
|
||||
"compilerOptions": {
|
||||
"inlineSourceMap": true,
|
||||
"inlineSources": true,
|
||||
"module": "ESNext",
|
||||
"target": "ES6",
|
||||
"allowJs": true,
|
||||
"noImplicitAny": true,
|
||||
"moduleResolution": "bundler",
|
||||
"importHelpers": true,
|
||||
"isolatedModules": true,
|
||||
"strictNullChecks": true,
|
||||
"lib": [
|
||||
"DOM",
|
||||
"ES5",
|
||||
"ES6",
|
||||
"ES7"
|
||||
]
|
||||
},
|
||||
"include": [
|
||||
"src/**/*.ts"
|
||||
]
|
||||
}
|
||||
14
version-bump.mjs
Normal file
14
version-bump.mjs
Normal file
|
|
@ -0,0 +1,14 @@
|
|||
import { readFileSync, writeFileSync } from "fs";
|
||||
|
||||
const targetVersion = process.env.npm_package_version;
|
||||
|
||||
// read minAppVersion from manifest.json and bump version to target version
|
||||
let manifest = JSON.parse(readFileSync("manifest.json", "utf8"));
|
||||
const { minAppVersion } = manifest;
|
||||
manifest.version = targetVersion;
|
||||
writeFileSync("manifest.json", JSON.stringify(manifest, null, "\t"));
|
||||
|
||||
// update versions.json with target version and minAppVersion from manifest.json
|
||||
let versions = JSON.parse(readFileSync("versions.json", "utf8"));
|
||||
versions[targetVersion] = minAppVersion;
|
||||
writeFileSync("versions.json", JSON.stringify(versions, null, "\t"));
|
||||
3
versions.json
Normal file
3
versions.json
Normal file
|
|
@ -0,0 +1,3 @@
|
|||
{
|
||||
"0.1.0": "0.15.0"
|
||||
}
|
||||
Loading…
Reference in a new issue