// [START functions_http_form_data] /** * Parses a 'multipart/form-data' upload request * * @param {Object} req Cloud Function request context. * @param {Object} res Cloud Function response context. */ const path = require('path'); const Arweave = require('arweave'); const { Storage } = require('@google-cloud/storage'); const os = require('os'); const fs = require('fs'); const crypto = require('crypto'); const { Account, Connection } = require('@solana/web3.js'); const mimeType = require('mime-types'); const fetch = require('node-fetch'); const storage = new Storage(); const BUCKET_NAME = 'us.artifacts.principal-lane-200702.appspot.com'; const FOLDER_NAME = 'arweave'; const ARWEAVE_KEYNAME = 'arweave.json'; const SOLANA_KEYNAME = 'arweave-sol-container.json'; const CLUSTER = 'https://devnet.solana.com'; //const CLUSTER = 'https://api.mainnet-beta.solana.com'; const SYSTEM = '11111111111111111111111111111111'; const MEMO = 'MemoSq4gqABAXKb96qnH8TysNcWxMyWCqXgDLGmfcHr'; const KEYHOLDER = {}; const FAIL = 'fail'; const SUCCESS = 'success'; const LAMPORT_MULTIPLIER = 10 ** 9; const WINSTON_MULTIPLIER = 10 ** 12; const RESERVED_TXN_MANIFEST = 'manifest.json'; function generateManifest(pathMap, indexPath) { const manifest = { manifest: 'arweave/paths', version: '0.1.0', paths: pathMap, }; if (indexPath) { if (!Object.keys(pathMap).includes(indexPath)) { throw new Error( `--index path not found in directory paths: ${indexPath}`, ); } manifest.index = { path: indexPath, }; } return manifest; } const getKey = async function (name) { if (KEYHOLDER[name]) return KEYHOLDER[name]; const options = { destination: os.tmpdir() + '/' + name, }; // Downloads the file await storage .bucket(BUCKET_NAME) .file(FOLDER_NAME + '/' + name) .download(options); console.log(`Key downloaded to ${os.tmpdir()}/${name}`); let rawdata = fs.readFileSync(os.tmpdir() + '/' + name); let key; try { key = JSON.parse(rawdata); } catch (e) { key = rawdata.toString(); } KEYHOLDER[name] = key; return KEYHOLDER[name]; }; // Node.js doesn't have a built-in multipart/form-data parsing library. // Instead, we can use the 'busboy' library from NPM to parse these requests. const Busboy = require('busboy'); const arweaveConnection = Arweave.init({ host: 'arweave.net', // Hostname or IP address for a Arweave host port: 443, // Port protocol: 'https', // Network protocol http or https timeout: 20000, // Network request timeouts in milliseconds logging: true, // Enable network request logging }); // FYI no streaming uploads as yet // https://gist.github.com/CDDelta/e2af7e02314b2e0c3b5f9eb616c645a6 // Need to read entire thing into memory - Limits us to 2GB files. TODO come back and implemnet. exports.uploadFile = async (req, res) => { res.set('Access-Control-Allow-Origin', '*'); if (req.method === 'OPTIONS') { // Send response to OPTIONS requests res.set('Access-Control-Allow-Methods', 'POST'); res.set('Access-Control-Allow-Headers', 'Content-Type'); res.set('Access-Control-Max-Age', '3600'); res.status(204).send(''); return; } if (req.method !== 'POST') { // Return a "method not allowed" error return res.status(405).end(); } const solanaKey = await getKey(SOLANA_KEYNAME); const solanaConnection = new Connection(CLUSTER, 'recent'); const solanaWallet = new Account(solanaKey); const arweaveWallet = await getKey(ARWEAVE_KEYNAME); console.log('Connections established.'); const busboy = new Busboy({ headers: req.headers }); const tmpdir = os.tmpdir(); const fieldPromises = []; // This code will process each non-file field in the form. busboy.on('field', (fieldname, val) => { console.log('I see ' + fieldname); fieldPromises.push( new Promise(async (res, _) => { if (fieldname === 'transaction') { try { console.log('Calling out for txn', val); const transaction = await solanaConnection.getParsedConfirmedTransaction( val, ); console.log('I got the transaction'); // We expect the first command to be a SOL send from them to our holding account. // Then after that it's memos of sha256 hashes of file contents. const expectedSend = transaction.transaction.message.instructions[0]; const isSystem = expectedSend.programId.toBase58() === SYSTEM; const isToUs = expectedSend.parsed.info.destination === solanaWallet.publicKey.toBase58(); console.log( 'Expected to send is', JSON.stringify(expectedSend.parsed), ); if (isSystem && isToUs) { const amount = expectedSend.parsed.info.lamports; const remainingMemos = transaction.transaction.message.instructions.filter( i => i.programId.toBase58() === MEMO, ); const memoMessages = remainingMemos.map(m => m.parsed); res({ name: fieldname, amount, memoMessages, }); } else throw new Error( 'No payment found because either the program wasnt the system program or it wasnt to the holding account', ); } catch (e) { console.log(fieldname, e); console.log('Setting txn anyway'); res({ name: fieldname, amount: 0, memoMessages: [], }); } } else if (fieldname === 'tags') { try { res({ name: fieldname, ...JSON.parse(val), }); } catch (e) { console.log(fieldname, e); res({ name: fieldname, }); } } }), ); }); const fileWrites = []; // This code will process each file uploaded. busboy.on('file', (fieldname, file, filename) => { // Note: os.tmpdir() points to an in-memory file system on GCF // Thus, any files in it must fit in the instance's memory. console.log(`Processed file ${filename}`); const filepath = path.join(tmpdir, filename); const writeStream = fs.createWriteStream(filepath); file.pipe(writeStream); // File was processed by Busboy; wait for it to be written. // Note: GCF may not persist saved files across invocations. // Persistent files must be kept in other locations // (such as Cloud Storage buckets). const promise = new Promise((resolve, reject) => { file.on('end', () => { writeStream.end(); }); writeStream.on('finish', resolve({ status: SUCCESS, filepath })); writeStream.on( 'error', reject({ status: FAIL, filepath, error: 'failed to save' }), ); }); fileWrites.push(promise); }); // Triggered once all uploaded files are processed by Busboy. // We still need to wait for the disk writes (saves) to complete. const body = { messages: [] }; busboy.on('finish', async () => { console.log('Finish'); const filepaths = [ ...(await Promise.all(fileWrites)), { filepath: RESERVED_TXN_MANIFEST, status: SUCCESS }, ]; const fields = await Promise.all(fieldPromises); const anchor = (await arweaveConnection.api.get('tx_anchor')).data; console.log('The one guy is ' + fields.map(f => f.name).join(',')); const txn = fields.find(f => f.name === 'transaction'); const fieldTags = fields.find(f => f.name === 'tags'); if (!txn || !txn.amount) { body.error = 'No transaction found with payment'; res.end(JSON.stringify(body)); return; } let runningTotal = txn.amount; const conversionRates = JSON.parse( await ( await fetch( 'https://api.coingecko.com/api/v3/simple/price?ids=solana,arweave&vs_currencies=usd', ) ).text(), ); // To figure out how much solana is required, multiply ar byte cost by this number const arMultiplier = conversionRates.arweave.usd / conversionRates.solana.usd; const paths = {}; for (let i = 0; i < filepaths.length; i++) { const f = filepaths[i]; if (f.status == FAIL) { body.messages.push(f); } else { const { filepath } = f; const parts = filepath.split('/'); const filename = parts[parts.length - 1]; try { let data, fileSizeInBytes, mime; if (filepath == RESERVED_TXN_MANIFEST) { const manifest = await generateManifest(paths, 'metadata.json'); data = Buffer.from(JSON.stringify(manifest), 'utf8'); fileSizeInBytes = data.byteLength; mime = 'application/x.arweave-manifest+json'; } else { data = fs.readFileSync(filepath); // Have to get separate Buffer since buffers are stateful const hashSum = crypto.createHash('sha256'); hashSum.update(data.toString()); const hex = hashSum.digest('hex'); if (!txn.memoMessages.find(m => m === hex)) { body.messages.push({ filename, status: FAIL, error: `Unable to find proof that you paid for this file, your hash is ${hex}, comparing to ${txn.memoMessages.join( ',', )}`, }); continue; } const stats = fs.statSync(filepath); fileSizeInBytes = stats.size; mime = mimeType.lookup(filepath); } const costSizeInWinstons = parseInt( await ( await fetch( 'https://arweave.net/price/' + fileSizeInBytes.toString(), ) ).text(), ); const costToStoreInSolana = (costSizeInWinstons * arMultiplier) / WINSTON_MULTIPLIER; runningTotal -= costToStoreInSolana * LAMPORT_MULTIPLIER; if (runningTotal > 0) { const transaction = await arweaveConnection.createTransaction( { data: data, last_tx: anchor }, arweaveWallet, ); transaction.addTag('Content-Type', mime); if (fieldTags) { const tags = fieldTags[filepath.split('/')[filepath.split('/').length - 1]]; if (tags) tags.map(t => transaction.addTag(t.name, t.value)); } await arweaveConnection.transactions.sign( transaction, arweaveWallet, ); await arweaveConnection.transactions.post(transaction); body.messages.push({ filename, status: SUCCESS, transactionId: transaction.id, }); paths[filename] = { id: transaction.id }; } else { body.messages.push({ filename, status: FAIL, error: `Not enough funds provided to push this file, you need at least ${costToStoreInSolana} SOL or ${costSize} AR`, }); } } catch (e) { console.log(e); body.messages.push({ filename, status: FAIL, error: e.toString() }); } } } res.end(JSON.stringify(body)); }); busboy.end(req.rawBody); }; // [END functions_http_form_data]