diff --git a/packages/arweave-push/.gcloudignore b/packages/arweave-push/.gcloudignore new file mode 100644 index 0000000..0f9ec0f --- /dev/null +++ b/packages/arweave-push/.gcloudignore @@ -0,0 +1,16 @@ +# This file specifies files that are *not* uploaded to Google Cloud Platform +# using gcloud. It follows the same syntax as .gitignore, with the addition of +# "#!include" directives (which insert the entries of the given .gitignore-style +# file at that point). +# +# For more information, run: +# $ gcloud topic gcloudignore +# +.gcloudignore +# If you would like to upload your .git directory, .gitignore file or files +# from your .gitignore file, remove the corresponding line +# below: +.git +.gitignore + +node_modules \ No newline at end of file diff --git a/packages/arweave-push/README b/packages/arweave-push/README new file mode 100644 index 0000000..137e283 --- /dev/null +++ b/packages/arweave-push/README @@ -0,0 +1 @@ +gcloud functions deploy uploadFile --runtime nodejs12 --trigger-http --allow-unauthenticated \ No newline at end of file diff --git a/packages/arweave-push/index.js b/packages/arweave-push/index.js new file mode 100644 index 0000000..94fbe5e --- /dev/null +++ b/packages/arweave-push/index.js @@ -0,0 +1,314 @@ +// [START functions_http_form_data] +/** + * Parses a 'multipart/form-data' upload request + * + * @param {Object} req Cloud Function request context. + * @param {Object} res Cloud Function response context. + */ +const path = require('path'); +const Arweave = require('arweave'); +const { Storage } = require('@google-cloud/storage'); +const os = require('os'); +const fs = require('fs'); +const crypto = require('crypto'); +const { Account, Connection } = require('@solana/web3.js'); +const mimeType = require('mime-types'); +const fetch = require('node-fetch'); + +const storage = new Storage(); +const BUCKET_NAME = 'us.artifacts.principal-lane-200702.appspot.com'; +const FOLDER_NAME = 'arweave'; +const ARWEAVE_KEYNAME = 'arweave.json'; +const SOLANA_KEYNAME = 'arweave-sol-container.json'; +const CLUSTER = 'https://devnet.solana.com'; +const SYSTEM = '11111111111111111111111111111111'; +const MEMO = 'MemoSq4gqABAXKb96qnH8TysNcWxMyWCqXgDLGmfcHr'; +const KEYHOLDER = {}; +const FAIL = 'fail'; +const SUCCESS = 'success'; + +const getKey = async function (name) { + if (KEYHOLDER[name]) return KEYHOLDER[name]; + + const options = { + destination: os.tmpdir() + '/' + name, + }; + + // Downloads the file + await storage + .bucket(BUCKET_NAME) + .file(FOLDER_NAME + '/' + name) + .download(options); + + console.log(`Key downloaded to ${os.tmpdir()}/${name}`); + + let rawdata = fs.readFileSync(os.tmpdir() + '/' + name); + let key; + try { + key = JSON.parse(rawdata); + } catch (e) { + key = rawdata.toString(); + } + + KEYHOLDER[name] = key; + return KEYHOLDER[name]; +}; + +// Node.js doesn't have a built-in multipart/form-data parsing library. +// Instead, we can use the 'busboy' library from NPM to parse these requests. +const Busboy = require('busboy'); +const arweaveConnection = Arweave.init({ + host: 'arweave.net', // Hostname or IP address for a Arweave host + port: 443, // Port + protocol: 'https', // Network protocol http or https + timeout: 20000, // Network request timeouts in milliseconds + logging: true, // Enable network request logging +}); + +// FYI no streaming uploads as yet +// https://gist.github.com/CDDelta/e2af7e02314b2e0c3b5f9eb616c645a6 +// Need to read entire thing into memory - Limits us to 2GB files. TODO come back and implemnet. +exports.uploadFile = async (req, res) => { + res.set('Access-Control-Allow-Origin', '*'); + + if (req.method === 'OPTIONS') { + // Send response to OPTIONS requests + res.set('Access-Control-Allow-Methods', 'POST'); + res.set('Access-Control-Allow-Headers', 'Content-Type'); + res.set('Access-Control-Max-Age', '3600'); + res.status(204).send(''); + return; + } + + if (req.method !== 'POST') { + // Return a "method not allowed" error + return res.status(405).end(); + } + const solanaKey = await getKey(SOLANA_KEYNAME); + const solanaConnection = new Connection(CLUSTER, 'recent'); + const solanaWallet = new Account(solanaKey); + const arweaveWallet = await getKey(ARWEAVE_KEYNAME); + console.log('Connections established.'); + const busboy = new Busboy({ headers: req.headers }); + const tmpdir = os.tmpdir(); + + const fieldPromises = []; + + // This code will process each non-file field in the form. + busboy.on('field', (fieldname, val) => { + console.log('I see ' + fieldname); + fieldPromises.push( + new Promise(async (res, _) => { + if (fieldname === 'transaction') { + try { + console.log('Calling out for txn', val); + const transaction = await solanaConnection.getParsedConfirmedTransaction( + val, + ); + console.log('I got the transaction'); + // We expect the first command to be a SOL send from them to our holding account. + // Then after that it's memos of sha256 hashes of file contents. + const expectedSend = + transaction.transaction.message.instructions[0]; + + const isSystem = expectedSend.programId.toBase58() === SYSTEM; + const isToUs = + expectedSend.parsed.info.destination === + solanaWallet.publicKey.toBase58(); + console.log( + 'Expected to send is', + JSON.stringify(expectedSend.parsed), + ); + if (isSystem && isToUs) { + const amount = expectedSend.parsed.info.lamports; + const remainingMemos = transaction.transaction.message.instructions.filter( + i => i.programId.toBase58() === MEMO, + ); + const memoMessages = remainingMemos.map(m => m.parsed); + res({ + name: fieldname, + amount, + memoMessages, + }); + } else + throw new Error( + 'No payment found because either the program wasnt the system program or it wasnt to the holding account', + ); + } catch (e) { + console.log(fieldname, e); + console.log('Setting txn anyway'); + res({ + name: fieldname, + amount: 0, + memoMessages: [], + }); + } + } else if (fieldname === 'tags') { + try { + res({ + name: fieldname, + ...JSON.parse(val), + }); + } catch (e) { + console.log(fieldname, e); + res({ + name: fieldname, + }); + } + } + }), + ); + }); + + const fileWrites = []; + + // This code will process each file uploaded. + busboy.on('file', (fieldname, file, filename) => { + // Note: os.tmpdir() points to an in-memory file system on GCF + // Thus, any files in it must fit in the instance's memory. + console.log(`Processed file ${filename}`); + const filepath = path.join(tmpdir, filename); + + const writeStream = fs.createWriteStream(filepath); + file.pipe(writeStream); + + // File was processed by Busboy; wait for it to be written. + // Note: GCF may not persist saved files across invocations. + // Persistent files must be kept in other locations + // (such as Cloud Storage buckets). + const promise = new Promise((resolve, reject) => { + file.on('end', () => { + writeStream.end(); + }); + writeStream.on('finish', resolve({ status: SUCCESS, filepath })); + writeStream.on( + 'error', + reject({ status: FAIL, filepath, error: 'failed to save' }), + ); + }); + + fileWrites.push(promise); + }); + + // Triggered once all uploaded files are processed by Busboy. + // We still need to wait for the disk writes (saves) to complete. + const body = { messages: [] }; + + busboy.on('finish', async () => { + console.log('Finish'); + const filepaths = await Promise.all(fileWrites); + const fields = await Promise.all(fieldPromises); + + console.log('The one guy is ' + fields.map(f => f.name).join(',')); + const txn = fields.find(f => f.name === 'transaction'); + const fieldTags = fields.find(f => f.name === 'tags'); + + if (!txn || !txn.amount) { + body.error = 'No transaction found with payment'; + return; + } + + let runningTotal = txn.amount; + + const conversionRates = JSON.parse( + await ( + await fetch( + 'https://api.coingecko.com/api/v3/simple/price?ids=solana,arweave&vs_currencies=usd', + ) + ).text(), + ); + + // To figure out how much solana is required, multiply ar byte cost by this number + const arMultiplier = + conversionRates.arweave.usd / conversionRates.solana.usd; + + filepaths.forEach(async f => { + if (f.status == FAIL) { + body.messages.push(f); + } else { + const { filepath } = f; + const parts = filepath.split('/'); + const filename = parts[parts.length - 1]; + try { + const data = fs.readFileSync(filepath); + // Have to get separate Buffer since buffers are stateful + const cryptoBuffer = fs.readFileSync(filepath); + const hashSum = crypto.createHash('sha256'); + hashSum.update(cryptoBuffer); + const hex = hashSum.digest('hex'); + + if (!fields.transaction.memoMessages.find(m => m === hex)) { + body.messages.push({ + filename, + status: FAIL, + error: 'Unable to find proof that you paid for this file', + }); + } + + const stats = fs.statSync(filepath); + const fileSizeInBytes = stats.size; + console.log(`File size ${fileSizeInBytes}`); + + const mime = mimeType.lookup(filepath); + + const costSize = parseInt( + await ( + await fetch( + 'https://arweave.net/price/' + fileSizeInBytes.toString(), + ) + ).text(), + ); + + const costToStoreInSolana = costSize * arMultiplier; + runningTotal -= costToStoreInSolana; + if (runningTotal > 0) { + console.log('My arweave wallet is ', arweaveWallet); + const transaction = await arweaveConnection.createTransaction( + { data: data }, + arweaveWallet, + ); + transaction.addTag('Content-Type', mime); + if (fieldTags) { + const tags = + fieldTags[filepath.split('/')[filepath.split('/').length - 1]]; + if (tags) tags.map(t => transaction.addTag(t.name, t.value)); + } + + await arweaveConnection.transactions.sign( + transaction, + arweaveWallet, + ); + + let uploader = await arweaveConnection.transactions.getUploader( + transaction, + ); + + while (!uploader.isComplete) { + await uploader.uploadChunk(); + console.log( + `${uploader.pctComplete}% complete, ${uploader.uploadedChunks}/${uploader.totalChunks}`, + ); + } + body.messages.push({ + filename, + status: SUCCESS, + transactionId: transaction.id, + }); + } else { + body.messages.push({ + filename, + status: FAIL, + error: `Not enough funds provided to push this file, you need at least ${costToStoreInSolana} SOL or ${costSize} AR`, + }); + } + } catch (e) { + console.log(e); + body.messages.push({ filename, status: FAIL, error: e.toString() }); + } + } + res.end(JSON.stringify(body)); + }); + }); + busboy.end(req.rawBody); +}; +// [END functions_http_form_data] diff --git a/packages/arweave-push/package.json b/packages/arweave-push/package.json new file mode 100644 index 0000000..022edd7 --- /dev/null +++ b/packages/arweave-push/package.json @@ -0,0 +1,32 @@ +{ + "name": "nodejs-docs-samples-functions-http", + "version": "0.0.1", + "private": true, + "license": "Apache-2.0", + "author": "Google Inc.", + "repository": { + "type": "git", + "url": "https://github.com/GoogleCloudPlatform/nodejs-docs-samples.git" + }, + "engines": { + "node": ">=12.0.0" + }, + "scripts": { + "test": "mocha test/*.test.js --timeout=60000" + }, + "devDependencies": { + "mocha": "^8.0.0", + "proxyquire": "^2.1.0", + "sinon": "^10.0.0" + }, + "dependencies": { + "@google-cloud/storage": "^5.0.0", + "busboy": "^0.3.0", + "escape-html": "^1.0.3", + "arweave": "1.10.13", + "@solana/web3.js": "^0.86.2", + "mime-types": "2.1.30", + "node-fetch": "2.6.1", + "coingecko-api": "1.0.10" + } +}