kubo in api

README.md

@@ -22,7 +22,7 @@ In a canvas, select a node and run hit Ctrl-P to search for
 - runCowsay - execute cowsay on the content of a canvas node
 - getDpid - pull a desci node from beta.dpid.org
 - runSDXL - run stable diffusion by executing a transation through lilypad
-- ipfsCAT - pull the content from a CID from ipfs
+- ipfsCat - pull the content from a CID from ipfs
 - ipfsDagGet - pull a cid that references a dag and splay the child CIds into nodes
 - ipfsAdd - Add a json object referenced by a CID
 

api/Dockerfile

@@ -1,7 +1,11 @@
 FROM node:20.7-buster
-
-
 WORKDIR /app
+RUN wget https://dist.ipfs.tech/kubo/v0.22.0/kubo_v0.22.0_linux-amd64.tar.gz
+RUN tar -xvzf kubo_v0.22.0_linux-amd64.tar.gz
+RUN cd kubo
+#RUN bash install.sh
+#RUN ipfs init
+#RUN ipfs daemon
 
 # get working with macos
 RUN curl -s "https://get.kamu.dev" | sh

api/gateway.js

@@ -0,0 +1,119 @@
+import pkg from 'js-sha3'
+const { keccak256 } = pkg
+import { from } from 'multiformats/hashes/hasher'
+import * as sha2 from 'multiformats/hashes/sha2'
+
+export default async function getHasherForCode(code) {
+    switch (code) {
+        case sha2.sha256.code:
+            return sha2.sha256
+        case sha2.sha512.code:
+            return sha2.sha512
+        case 17:
+            // git hasher uses sha1. see ipld-git/src/util.js
+            return from({
+                name: 'sha1',
+                code,
+                encode: async (data) => {
+                    const crypto = globalThis.crypto ?? (await import('crypto')).webcrypto
+                    const hashBuffer = await crypto.subtle.digest('SHA-1', data)
+                    return new Uint8Array(hashBuffer)
+                }
+            })
+        case 27: // keccak-256
+            return from({
+                name: 'keccak-256',
+                code,
+                encode: async (data) => {
+                    return new Uint8Array(keccak256.arrayBuffer(data))
+                }
+            })
+
+        default:
+            throw new Error(`unknown multihasher code '${code}'`)
+    }
+}
+
+
+async function getCidFromBytes(bytes, cidVersion, codecCode, multihashCode) {
+    const hasher = await getHasherForCode(multihashCode)
+
+    try {
+        const hash = await hasher.digest(bytes)
+        return CID.create(cidVersion, codecCode, hash)
+    } catch (err) {
+        console.error('could not create cid from bytes', err)
+    }
+
+    return ''
+}
+export async function verifyBytes(providedCid, bytes) {
+    try {
+        const cid = await getCidFromBytes(bytes, providedCid.version, providedCid.code, providedCid.multihash.code)
+
+        if (cid.toString() !== providedCid.toString()) {
+            throw new Error(`CID mismatch, expected '${providedCid.toString()}' but got '${cid.toString()}'`)
+        }
+    } catch (err) {
+        console.error('unable to verify bytes', err)
+        throw err
+    }
+}
+
+function ensureGatewayFetchEnabled() {
+    console.info('import.meta.env.NODE_ENV: ', import.meta.env.NODE_ENV)
+    console.info(
+        "🎛️ Customise whether ipld-explorer-components fetches content fro gateways by setting an `explore.ipld.gatewayEnabled` value to true/false in localStorage. e.g. localStorage.setItem('explore.ipld.gatewayEnabled', false) -- NOTE: defaults to true"
+    )
+    const gatewayEnabledSetting = localStorage.getItem('explore.ipld.gatewayEnabled')
+
+    return gatewayEnabledSetting != null ? JSON.parse(gatewayEnabledSetting) : true
+}
+
+async function getRawBlockFromGateway (url, cid, signal) {
+    const gwUrl = new URL(url)
+    console.log('hi1', signal)
+    gwUrl.pathname = `/ipfs/${cid.toString()}`
+    gwUrl.search = '?format=raw' // necessary as not every gateway supports dag-cbor, but every should support sending raw block as-is
+    try {
+      const res = await fetch(gwUrl.toString(), {
+        headers: {
+          // also set header, just in case ?format= is filtered out by some reverse proxy
+          Accept: 'application/vnd.ipld.raw'
+        },
+        cache: 'force-cache'
+      })
+      if (!res.ok) {
+        throw new Error(`unable to fetch raw block for CID ${cid} from gateway ${gwUrl.toString()}`)
+      }
+      return new Uint8Array(await res.arrayBuffer())
+    } catch (cause) {
+      console.error('cause', cause)
+      throw new Error(`unable to fetch raw block for CID ${cid}`)
+    }
+  }
+export async function getBlockFromAnyGateway(cid, signal, moreGateways = []) {
+    const gateways = moreGateways.concat(defaultGateways)
+    for (const url of gateways) {
+        if (signal.aborted) {
+            throw new Error('aborted')
+        }
+        try {
+            console.log('hello')
+            const rawBlock = await getRawBlockFromGateway(url, cid, signal)
+            try {
+                await verifyBytes(cid, rawBlock)
+                return rawBlock
+            } catch (err) {
+                console.error('unable to verify block from gateway', url)
+                continue
+            }
+        } catch (err) {
+            console.error('unable to get block from gateway', err)
+            // ignore the error
+        }
+    }
+    throw new Error('Could not get block from any gateway')
+}
+
+const defaultGateways = ['https://ipfs.io', 'https://dweb.link']

api/index.js

@@ -6,12 +6,10 @@ import { CID } from 'multiformats/cid'
 import { createHelia } from 'helia'
 import { dagJson } from '@helia/dag-json'
 import { dagCbor } from '@helia/dag-cbor'
-
+import cors from '@fastify/cors'
 import Fastify from 'fastify'
-import {initLibp2p} from './kubo.js'
 import debug from 'debug'
 // debug.enable('helia:*,libp2p:*')
-
 import * as IpldDagJson from '@ipld/dag-json'
 import * as IpldDagPB from '@ipld/dag-pb'
 import * as IpldDagCbor from '@ipld/dag-cbor'
@@ -19,6 +17,11 @@ import {dagPb} from './helia-dag-pb.js'
 import { unixfs } from '@helia/unixfs'
 import { sha256 } from 'multiformats/hashes/sha2'
 
+import {initLibp2p} from './kubo.js'
+import {
+   getBlockFromAnyGateway,
+
+ } from './gateway.js';
 
 let heliaNode;
 let dJson;
@@ -34,6 +37,11 @@ const fastify = Fastify({
     logger: true
 })
 
+await fastify.register(cors, { 
+  // put your options here
+  origin: 'true'
+})
+
 // @ts-check
 const getOptions = {
   schema: {
@@ -47,6 +55,37 @@ const getOptions = {
 const mapFromCidCodeToHeliaWrapper = new Map()
 
 
+fastify.post('/gateway', getOptions, async (request, reply) => {
+  console.log(`request.query: `, request.query);
+  if (request.query.cid === undefined) {
+    reply.type('application/json').code(500)
+    return { error: 'no cid provided' }
+  }
+  let Cid = CID.parse(request.query.cid, sha256.decoder)
+  try {
+    //const block = await getBlockFromAnyGateway(Cid, new AbortController())
+    const res = fetch(`https://0.0.0.0:5001/api/v0/dag/get/${request.query.cid}`, {
+      method: 'post',
+      mode: 'no-cors',
+      headers: {
+        "Content-Type":"text/plain",
+        "Access-Control-Allow-Origin": "*",
+        "Access-Control-Allow-Headers":"X-Stream-Output, X-Chunked-Output, X-Content-Length",
+        "Access-Control-Expose-Headers": "X-Stream-Output, X-Chunked-Output, X-Content-Length",
+      }
+    }).then((res) => {
+      console.log('res', res)
+    })
+    console.log('res', res)
+  } catch (e) {
+    console.log('error:', e)
+    reply.type('application/json').code(500)
+    return { 'error': e}
+  }
+
+})
+
+
 fastify.get('/', getOptions, async (request, reply) => {
   // console.log(request.query)
   console.log(`request.query: `, request.query);

api/kubo/LICENSE

@@ -0,0 +1,8 @@
+This project is transitioning from an MIT-only license to a dual MIT/Apache-2.0 license. 
+Unless otherwise noted, all code contributed prior to 2019-05-06 and not contributed by 
+a user listed in [this signoff issue](https://github.com/ipfs/go-ipfs/issues/6302) is 
+licensed under MIT-only. All new contributions (and past contributions since 2019-05-06) 
+are licensed under a dual MIT/Apache-2.0 license.
+
+MIT: https://www.opensource.org/licenses/mit
+Apache-2.0: https://www.apache.org/licenses/license-2.0

api/kubo/LICENSE-APACHE

@@ -0,0 +1,5 @@
+Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.

api/kubo/LICENSE-MIT

@@ -0,0 +1,19 @@
+The MIT License (MIT)
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

api/kubo/README.md

@@ -0,0 +1,28 @@
+# ipfs command line tool
+
+This is the [ipfs](http://ipfs.io) command line tool. It contains a full ipfs node.
+
+## Install
+
+To install it, move the binary somewhere in your `$PATH`:
+
+```sh
+sudo mv ipfs /usr/local/bin/ipfs
+```
+
+Or run `sudo ./install.sh` which does this for you.
+
+## Usage
+
+First, you must initialize your local ipfs node:
+
+```sh
+ipfs init
+```
+
+This will give you directions to get started with ipfs.
+You can always get help with:
+
+```sh
+ipfs --help
+```

api/kubo/install.sh

@@ -0,0 +1,39 @@
+#!/bin/sh
+#
+# Installation script for ipfs. It tries to move $bin in one of the
+# directories stored in $binpaths.
+
+INSTALL_DIR=$(dirname $0)
+
+bin="$INSTALL_DIR/ipfs"
+binpaths='/usr/local/bin /usr/bin $HOME/.local/bin'
+
+# This variable contains a nonzero length string in case the script fails
+# because of missing write permissions.
+is_write_perm_missing=""
+
+for raw in $binpaths; do
+  # Expand the $HOME variable.
+  binpath=$(eval echo "$raw")
+  mkdir -p "$binpath"
+  if mv "$bin" "$binpath/ipfs" ; then
+    echo "Moved $bin to $binpath"
+    exit 0
+  else
+    if [ -d "$binpath" ] && [ ! -w "$binpath" ]; then
+      is_write_perm_missing=1
+    fi
+  fi
+done
+
+echo "We cannot install $bin in one of the directories $binpaths"
+
+if [ -n "$is_write_perm_missing" ]; then
+  echo "It seems that we do not have the necessary write permissions."
+  echo "Perhaps try running this script as a privileged user:"
+  echo
+  echo "    sudo $0"
+  echo
+fi
+
+exit 1

api/package-lock.json

@@ -11,6 +11,7 @@
       "dependencies": {
         "@chainsafe/libp2p-noise": "^13.0.1",
         "@chainsafe/libp2p-yamux": "^5.0.0",
+        "@fastify/cors": "^8.4.0",
         "@helia/dag-cbor": "^1.0.2",
         "@helia/dag-json": "^1.0.2",
         "@helia/unixfs": "^1.4.2",
@@ -25,6 +26,7 @@
         "datastore-core": "^9.2.3",
         "fastify": "^4.23.2",
         "helia": "^2.0.2",
+        "js-sha3": "^0.9.2",
         "kubo-rpc-client": "^3.0.1",
         "libp2p": "^0.46.11",
         "multiformats": "^12.1.1"
@@ -1904,6 +1906,15 @@
         "fast-uri": "^2.0.0"
       }
     },
+    "node_modules/@fastify/cors": {
+      "version": "8.4.0",
+      "resolved": "https://registry.npmjs.org/@fastify/cors/-/cors-8.4.0.tgz",
+      "integrity": "sha512-MlVvMTenltToByTpLwlWtO+7dQ3l2J+1OpmGrx9JpSNWo1d+dhfNCOi23zHhxdFhtpDzfwGwCsKu9DTeG7k7nQ==",
+      "dependencies": {
+        "fastify-plugin": "^4.0.0",
+        "mnemonist": "0.39.5"
+      }
+    },
     "node_modules/@fastify/deepmerge": {
       "version": "1.3.0",
       "resolved": "https://registry.npmjs.org/@fastify/deepmerge/-/deepmerge-1.3.0.tgz",
@@ -4224,6 +4235,11 @@
         "toad-cache": "^3.2.0"
       }
     },
+    "node_modules/fastify-plugin": {
+      "version": "4.5.1",
+      "resolved": "https://registry.npmjs.org/fastify-plugin/-/fastify-plugin-4.5.1.tgz",
+      "integrity": "sha512-stRHYGeuqpEZTL1Ef0Ovr2ltazUT9g844X5z/zEBFLG8RYlpDiOCIG+ATvYEp+/zmc7sN29mcIMp8gvYplYPIQ=="
+    },
     "node_modules/fastify/node_modules/lru-cache": {
       "version": "6.0.0",
       "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz",
@@ -5847,6 +5863,11 @@
         "string_decoder": "^1.2.0"
       }
     },
+    "node_modules/js-sha3": {
+      "version": "0.9.2",
+      "resolved": "https://registry.npmjs.org/js-sha3/-/js-sha3-0.9.2.tgz",
+      "integrity": "sha512-8kgvwd03wNGQG1GRvl3yy1Yt40sICAcIMsDU2ZLgoL0Z6z9rkRmf9Vd+bi/gYSzgAqMUGl/jiDKu0J8AWFd+BQ=="
+    },
     "node_modules/js-tokens": {
       "version": "4.0.0",
       "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz",
@@ -6221,6 +6242,14 @@
         "url": "https://github.com/sponsors/ljharb"
       }
     },
+    "node_modules/mnemonist": {
+      "version": "0.39.5",
+      "resolved": "https://registry.npmjs.org/mnemonist/-/mnemonist-0.39.5.tgz",
+      "integrity": "sha512-FPUtkhtJ0efmEFGpU14x7jGbTB+s18LrzRL2KgoWz9YvcY3cPomz8tih01GbHwnGk/OmkOKfqd/RAQoc8Lm7DQ==",
+      "dependencies": {
+        "obliterator": "^2.0.1"
+      }
+    },
     "node_modules/mortice": {
       "version": "3.0.1",
       "resolved": "https://registry.npmjs.org/mortice/-/mortice-3.0.1.tgz",
@@ -7145,6 +7174,11 @@
         "url": "https://github.com/sponsors/ljharb"
       }
     },
+    "node_modules/obliterator": {
+      "version": "2.0.4",
+      "resolved": "https://registry.npmjs.org/obliterator/-/obliterator-2.0.4.tgz",
+      "integrity": "sha512-lgHwxlxV1qIg1Eap7LgIeoBWIMFibOjbrYPIPJZcI1mmGAI2m3lNYpK12Y+GBdPQ0U1hRwSord7GIaawz962qQ=="
+    },
     "node_modules/observable-webworkers": {
       "version": "2.0.1",
       "resolved": "https://registry.npmjs.org/observable-webworkers/-/observable-webworkers-2.0.1.tgz",

api/package.json

@@ -20,6 +20,7 @@
   "dependencies": {
     "@chainsafe/libp2p-noise": "^13.0.1",
     "@chainsafe/libp2p-yamux": "^5.0.0",
+    "@fastify/cors": "^8.4.0",
     "@helia/dag-cbor": "^1.0.2",
     "@helia/dag-json": "^1.0.2",
     "@helia/unixfs": "^1.4.2",
@@ -34,6 +35,7 @@
     "datastore-core": "^9.2.3",
     "fastify": "^4.23.2",
     "helia": "^2.0.2",
+    "js-sha3": "^0.9.2",
     "kubo-rpc-client": "^3.0.1",
     "libp2p": "^0.46.11",
     "multiformats": "^12.1.1"

docker-compose.dev.yaml

@@ -13,21 +13,6 @@ services:
       dockerfile: ./Dockerfile
     ports:
       - 3000:3000
-    volumes:
-      - ./api:/app
-      - /app/node_modules
-    command: npm run start
-
-  kubo:
-    image: ipfs/kubo:latest
-    restart: unless-stopped
-    volumes:
-      - ipfs_path:/data/ipfs
-      - ipfs_fuse:/ipfs
-      - ipns_fuse:/ipns
-    environment:
-      - IPFS_PATH=/data/ipfs
-    ports:
       # Swarm listens on all interfaces, so is remotely reachable.
       - 4001:4001/tcp
       - 4001:4001/udp
@@ -40,6 +25,13 @@ services:
 
       # HTTP Gateway
       - 127.0.0.1:8080:8080
+    volumes:
+      - ipfs_path:/data/ipfs
+      - ipfs_fuse:/ipfs
+      - ipns_fuse:/ipns
+      - ./api:/app
+      - /app/node_modules
+    command: npm run start
 
 volumes:
   ipfs_path:

hardhat/contracts/KamuCaller.sol

@@ -0,0 +1,103 @@
+// SPDX-License-Identifier: MIT
+pragma solidity >=0.8.19;
+import "hardhat/console.sol";
+import "@openzeppelin/contracts/access/Ownable.sol";
+import "https://github.com/bacalhau-project/lilypad-v0/blob/main/hardhat/contracts/LilypadEventsUpgradeable.sol";
+import "https://github.com/bacalhau-project/lilypad-v0/blob/main/hardhat/contracts/LilypadCallerInterface.sol";
+​
+/**
+    @notice An experimental contract for POC work to call Bacalhau jobs from FVM smart contracts
+*/
+contract KamuCallerV2 is LilypadCallerInterface, Ownable {
+    address public bridgeAddress;
+    LilypadEventsUpgradeable bridge;
+    uint256 public lilypadFee; //=30000000000000000;
+​
+    struct KamuImage {
+        string kamuManifest;
+        string ipfsResult;
+    }
+​
+    KamuImage[] public images;
+    mapping (uint => string) prompts;
+​
+    event NewImageGenerated(KamuImage image);
+​
+    constructor(address _bridgeContractAddress) {
+        console.log("Deploying Kamu contract");
+        bridgeAddress = _bridgeContractAddress;
+        bridge = LilypadEventsUpgradeable(_bridgeContractAddress);
+        uint fee = bridge.getLilypadFee();
+        lilypadFee = fee;
+    }
+​
+    function setBridgeAddress(address _newAddress) public onlyOwner {
+      bridgeAddress= _newAddress;
+    }
+​
+    function setLPEventsAddress(address _eventsAddress) public onlyOwner {
+        bridge = LilypadEventsUpgradeable(_eventsAddress);
+    }
+​
+    function getLilypadFee() external {
+        uint fee = bridge.getLilypadFee(); 
+        console.log("fee", fee);
+        lilypadFee = fee;
+    }
+​
+    // not recommended
+    function setLilypadFee(uint256 _fee) public onlyOwner {
+        require(_fee > 0, "Lilypad fee must be greater than 0");
+        lilypadFee = _fee;
+    }
+​
+    string constant specStart = '{'
+        '"Engine": "docker",'
+        '"Verifier": "noop",'
+        '"PublisherSpec": {"Type": "estuary"},'
+        '"Docker": {'
+        '"Image": "ghcr.io/kamu-data/kamu-base:latest-with-data",'
+        '"Entrypoint": ["python", "main.py", "--o", "./outputs", "--p", "';
+​
+    string constant specEnd =
+        '"]},'
+        '"Resources": {"CPU": "1"},'
+        '"Outputs": [{"Name": "outputs", "Path": "/outputs"}],'
+        '"Deal": {"Concurrency": 1}'
+        '}';
+​
+    
+    
+    function Kamu(string calldata _prompt) external payable {
+        require(msg.value >= lilypadFee, "Not enough to run Lilypad job");
+        // TODO: spec -> do proper json encoding, look out for quotes in _prompt
+        string memory spec = string.concat(specStart, _prompt, specEnd);
+        uint id = bridge.runLilypadJob{value: lilypadFee}(address(this), spec, uint8(LilypadResultType.CID));
+        require(id > 0, "job didn't return a value");
+        prompts[id] = _prompt;
+    }
+​
+    function allImages() public view returns (KamuImage[] memory) {
+        return images;
+    }
+​
+    function lilypadFulfilled(address _from, uint _jobId, LilypadResultType _resultType, string calldata _result) external override {
+        //need some checks here that it a legitimate result
+        require(_from == address(bridge)); //really not secure
+        require(_resultType == LilypadResultType.CID);
+​
+        KamuImage memory image = KamuImage({
+            ipfsResult: _result,
+            prompt: prompts[_jobId]
+        });
+        images.push(image);
+        emit NewImageGenerated(image);
+        delete prompts[_jobId];
+    }
+​
+    function lilypadCancelled(address _from, uint _jobId, string calldata _errorMsg) external override {
+        require(_from == address(bridge)); //really not secure
+        console.log(_errorMsg);
+        delete prompts[_jobId];
+    }
+}

kamu-pipeline-job/Dockerfile

@@ -1,2 +1,4 @@
 FROM ghcr.io/kamu-data/kamu-base:latest-with-data
+FROM ipfs/kubo:latest
+
 

main.tsx

@@ -406,6 +406,15 @@ export default class ObsidianLilypad extends Plugin {
 		if (node) {
 			await canvas.requestSave()
 			await sleep(200)
+			console.log('siblings', siblings)
+			let climb = true
+			while (climb) {
+				const siblings = canvas.getEdgesForNode(node)
+				siblings.forEach((n) => {
+					
+				})
+			}
+
 
 			const settings = this.settings
 
@@ -418,7 +427,7 @@ export default class ObsidianLilypad extends Plugin {
 
 			const created = createNode(canvas, node,
 				{
-					text: `adding ${nodeText} to ipfs`,
+					text: `attempting to convert tree to dag`,
 					size: { height: placeholderNoteHeight }
 				},
 				{
@@ -428,6 +437,16 @@ export default class ObsidianLilypad extends Plugin {
 			)
 
 			try {
+				/*
+				const dag = {}
+				const cid = await requestUrl({
+					url: `http://localhost:3000/add`,
+					method: 'POST',
+					body: JSON.stringify({
+						
+					})
+
+				})
 				const cid = await this.dagJsonInstance.add({
 					text: nodeText
 				})
@@ -441,7 +460,7 @@ export default class ObsidianLilypad extends Plugin {
 						chat_role: 'assistant'
 					}
 				)
-
+				*/
 			} catch (e) {
 				const cideNodeError = createNode(canvas, created,
 					{