From ca408f49d29ee457bfca8361a6a74230ba0ed4d6 Mon Sep 17 00:00:00 2001 From: EC2 Default User Date: Sat, 28 Oct 2017 23:31:03 +0000 Subject: [PATCH 1/3] ec2 --- database.js | 37 +++++++++++++++++++ index.js | 100 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 137 insertions(+) create mode 100644 database.js create mode 100644 index.js diff --git a/database.js b/database.js new file mode 100644 index 0000000..4c43569 --- /dev/null +++ b/database.js @@ -0,0 +1,37 @@ +const mysql = require('mysql'); +const config = require('./config.js'); + +const connection = mysql.createConnection({ + host: "rds-mysql-10mintutorial.c4b0uxv6yk5n.us-west-1.rds.amazonaws.com", + user: "masterUsername", + password: config.pw, + database: "realestate" +}); + +const insert = function(data) { + var query = data; + connection.query("INSERT INTO bitcoin_prices (nowtime, high, low, volume, last, ask, bid) VALUES (?, ?, ?, ?, ?, ?, ?)", [data.timestamp, data.high, data.low, data.volume, data.last, data.ask, data.bid], function(err, result) { + if (err) { + throw err; + } else { + console.log('Values Successfully Inserted!') + } + }) +}; + +const retrieve = function(query, callback) { + var query = query || "SELECT * FROM bitcoin_prices"; + connection.query(query, function(err, result) { + if (err) { + throw err; + } else { + console.log('Vales successfuly retrieved: ', result); + callback(result); + } + }); +}; + +module.exports = { + insert: insert, + retrieve: retrieve +}; \ No newline at end of file diff --git a/index.js b/index.js new file mode 100644 index 0000000..6fb9872 --- /dev/null +++ b/index.js @@ -0,0 +1,100 @@ +const express = require('express'); +const app = express(); +const mysql = require('mysql'); +const Exchanges = require('crypto-exchange'); +const cron = require('node-cron'); +const db = require('./database.js'); +const statsD = require('node-statsd'); + +const client = new statsD({ + host: `statsd.hostedgraphite.com`, + port: 8125, + prefix: `b9483767-cbb7-4699-8548-54f0245df2d5` +}) + +const _port = 3000; + +// MySql Database +// const connection = mysql.createConnection({ +// host: "rds-mysql-10mintutorial.c4b0uxv6yk5n.us-west-1.rds.amazonaws.com", +// user: "masterUsername", +// password: config.pw, +// database: "realestate" +// }); + +// connection.connect(function(err) { +// if (err) { +// console.log("Cannot Connect!", err); +// } else { +// console.log("Connected!"); +// } +// }) + +// connection.connect(function(err) { +// if (err) throw err; +// console.log("Connected!"); +// connection.query("DROP TABLE IF EXISTS bitcoin_prices", function(err, result) { +// connection.query(`CREATE TABLE bitcoin_prices ( +// nowtime BIGINT NOT NULL PRIMARY KEY, +// high INTEGER, +// low INTEGER, +// volume INTEGER, +// last INTEGER, +// ask INTEGER, +// bid INTEGER +// )`, function(err, result) { +// if (err) throw err; +// console.log('TABLE CREATED!'); + +// }); +// }); +// }); + + +// const insertMySql = function(data) { +// var query = data; +// con.query("INSERT INTO bitcoin_prices (nowtime, high, low, volume, last, ask, bid) VALUES (?, ?, ?, ?, ?, ?, ?)", [data.timestamp, data.high, data.low, data.volume, data.last, data.ask, data.bid], function(err, result) { +// if (err) { +// throw err; +// } else { +// console.log('Values Successfully Inserted!') +// } +// }) +// } + +// CREATE DATABASE IF NOT EXISTS real_estate_data; +// USE real_estate_data; +// DROP TABLE IF EXISTS bitcoin_prices; +// CREATE TABLE bitcoin_prices( +// nowtime BIGINT NOT NULL PRIMARY KEY, +// high INTEGER, +// low INTEGER, +// volume INTEGER, +// last INTEGER, +// ask INTEGER, +// bid INTEGER +// ); + + +// end MySql Database + +cron.schedule('*/1 * * * * *', function() { + Exchanges.kraken.ticker('BTC_USD') + .then(data => { + db.insert(data.BTC_USD); + client.increment('worker.bitcoin.insert'); + console.log('Values inserted: ', data.BTC_USD); + }) +}); + +// console.log(db.retrieve(null, console.log)); + +// make a cron schedule 1x/day to send all data from past 24 hrs to offline AI (new instance) +// + + + +app.get('/', (req, res) => { + res.send('JohnJohn Sucks') +}) +app.listen(_port, () => console.log('Server running on port ' + _port)) From 9a692aa9b841230b444ee687e4c7766804ec4e73 Mon Sep 17 00:00:00 2001 From: EC2 Default User Date: Wed, 1 Nov 2017 21:08:47 +0000 Subject: [PATCH 2/3] completes elatic IP and end point for gateway --- .gitignore | 2 + database.js | 30 ++++++++++-- index.js | 128 ++++++++++++++++++++++++++++++++++++++-------------- 3 files changed, 123 insertions(+), 37 deletions(-) diff --git a/.gitignore b/.gitignore index 00cbbdf..943677f 100644 --- a/.gitignore +++ b/.gitignore @@ -57,3 +57,5 @@ typings/ # dotenv environment variables file .env +# config +./config.js diff --git a/database.js b/database.js index 4c43569..18128b5 100644 --- a/database.js +++ b/database.js @@ -1,11 +1,20 @@ const mysql = require('mysql'); const config = require('./config.js'); +// BitCoin DB +// const connection = mysql.createConnection({ +// host: "rds-mysql-10mintutorial.c4b0uxv6yk5n.us-west-1.rds.amazonaws.com", +// user: "masterUsername", +// password: config.pw, +// database: "realestate" +// }); + +// House Prices DB const connection = mysql.createConnection({ - host: "rds-mysql-10mintutorial.c4b0uxv6yk5n.us-west-1.rds.amazonaws.com", - user: "masterUsername", + host: "house-prices.c4b0uxv6yk5n.us-west-1.rds.amazonaws.com", + user: "sevam", password: config.pw, - database: "realestate" + database: "closingHousePrices" }); const insert = function(data) { @@ -31,7 +40,20 @@ const retrieve = function(query, callback) { }); }; +const retrieveHomePrices = function(query, callback) { + var query = query || "SELECT * FROM sf_home_sale_prices"; + connection.query(query, function(err, result) { + if (err) { + throw err; + } else { + // console.log('Vales successfuly retrieved: ', result); + callback(result); + } + }); +}; + module.exports = { insert: insert, - retrieve: retrieve + retrieve: retrieve, + retrieveHomePrices: retrieveHomePrices }; \ No newline at end of file diff --git a/index.js b/index.js index 6fb9872..215cec9 100644 --- a/index.js +++ b/index.js @@ -5,21 +5,22 @@ const Exchanges = require('crypto-exchange'); const cron = require('node-cron'); const db = require('./database.js'); const statsD = require('node-statsd'); +// const config = require('./config.js') const client = new statsD({ host: `statsd.hostedgraphite.com`, port: 8125, prefix: `b9483767-cbb7-4699-8548-54f0245df2d5` -}) +}); -const _port = 3000; +const _port = 1337; // MySql Database // const connection = mysql.createConnection({ -// host: "rds-mysql-10mintutorial.c4b0uxv6yk5n.us-west-1.rds.amazonaws.com", -// user: "masterUsername", -// password: config.pw, -// database: "realestate" +// host: "house-prices.c4b0uxv6yk5n.us-west-1.rds.amazonaws.com", +// user: "sevam", +// database: "closingHousePrices", +// password: config.pw // }); // connection.connect(function(err) { @@ -33,22 +34,11 @@ const _port = 3000; // connection.connect(function(err) { // if (err) throw err; // console.log("Connected!"); -// connection.query("DROP TABLE IF EXISTS bitcoin_prices", function(err, result) { -// connection.query(`CREATE TABLE bitcoin_prices ( -// nowtime BIGINT NOT NULL PRIMARY KEY, -// high INTEGER, -// low INTEGER, -// volume INTEGER, -// last INTEGER, -// ask INTEGER, -// bid INTEGER -// )`, function(err, result) { -// if (err) throw err; -// console.log('TABLE CREATED!'); - +// connection.query("select * from sf_home_sale_prices where ZipCode=94102;", function(err, result) { +// console.log('result:',result); +// console.log(err) // }); // }); -// }); // const insertMySql = function(data) { @@ -78,23 +68,95 @@ const _port = 3000; // end MySql Database -cron.schedule('*/1 * * * * *', function() { - Exchanges.kraken.ticker('BTC_USD') - .then(data => { - db.insert(data.BTC_USD); - client.increment('worker.bitcoin.insert'); - console.log('Values inserted: ', data.BTC_USD); - }) -}); -// console.log(db.retrieve(null, console.log)); +// BitCoin Worker: +// cron.schedule('*/1 * * * * *', function() { +// Exchanges.kraken.ticker('BTC_USD') +// .then(data => { +// db.insert(data.BTC_USD); +// client.increment('worker.bitcoin.insert'); +// console.log('Values inserted: ', data.BTC_USD); +// }) +// }); + +// db.retrieveHomePrices("SELECT * FROM sf_home_sale_prices WHERE ZipCode=" + "94158", function(data) { +// var count = 0; +// var responseObj = {}; +// let startDate = "200410"; +// let endDate = "200510"; +// let sum = 0; +// for (var key in data[0]) { +// if (count > 0 || startDate === key) { +// if (key === endDate) { +// break; +// } else { +// count++; +// sum += data[0][key]; +// } +// } +// } +// let average = sum/count; +// if (average) { +// res.json({average: average}) +// } else { +// res.json({average: 1200000}) +// } +// }); // make a cron schedule 1x/day to send all data from past 24 hrs to offline AI (new instance) -// -app.get('/', (req, res) => { - res.send('JohnJohn Sucks') -}) +app.get('/*', (req, res) => { + console.log('Request coming in!', req.query) + + let zipcode = req.query.zipcode || "94118"; + let startDate = req.query.startDate.slice(0,4) + req.query.startDate.slice(5,7) || "200410"; + let endDate = req.query.endDate.slice(0,4) + req.query.endDate.slice(5,7) || "200510"; + db.retrieveHomePrices("SELECT * FROM sf_home_sale_prices WHERE ZipCode=" + zipcode, function(data) { + if (data) { + var count = 0; + var responseObj = {}; + let sum = 0; + for (var key in data[0]) { + if (count > 0 || startDate === key) { + if (key === endDate) { + break; + } else { + count++; + sum += data[0][key]; + } + } + } + let average = sum/count; + if (average) { + res.json({average: average}) + } else { + res.json({average: 1200000}) + } + } else { + res.json({error: 'Please specify the ZipCode'}) + } + }) +}); + + app.listen(_port, () => console.log('Server running on port ' + _port)) + + + + + + + + + + + + + + + + + + From 999e1f8928b349693dfcc3e54ec0873eadd75d80 Mon Sep 17 00:00:00 2001 From: EC2 Default User Date: Sat, 4 Nov 2017 18:24:54 +0000 Subject: [PATCH 3/3] completes load testing with 200 qps --- index.js | 88 +++++++++++++++++++++++++++++++++++++++----------------- 1 file changed, 62 insertions(+), 26 deletions(-) diff --git a/index.js b/index.js index 215cec9..c04f99f 100644 --- a/index.js +++ b/index.js @@ -5,15 +5,37 @@ const Exchanges = require('crypto-exchange'); const cron = require('node-cron'); const db = require('./database.js'); const statsD = require('node-statsd'); -// const config = require('./config.js') +const redis = require('redis'); -const client = new statsD({ +// telnet thesis-house-price-001.nbxzan.0001.usw1.cache.amazonaws.com 11211 +// src/redis-cli -c -h thesis-house-price.nbxzan.ng.0001.usw1.cache.amazonaws.com -p 6379 +const _port = 1337; + +const statsDClient = new statsD({ host: `statsd.hostedgraphite.com`, port: 8125, - prefix: `b9483767-cbb7-4699-8548-54f0245df2d5` + prefix: `00436c17-5dfb-4df2-bd21-634d9a0ab64f` }); -const _port = 1337; +const redisClient = redis.createClient({ + host: 'thesis-house-price.nbxzan.ng.0001.usw1.cache.amazonaws.com', + port: 6379 +}); + +redisClient.on("error", function (err) { + console.log("Error " + err); +}); + +// redisClient.set("string key", "string val", redis.print); +// redisClient.hset("hash key", 123, "some value", redis.print); +// redisClient.hset(["hash key", 444, "some other value"], redis.print); +// redisClient.hkeys("hash key", function (err, replies) { +// console.log(replies.length + " replies:"); +// replies.forEach(function (reply, i) { +// console.log(" " + i + ": " + reply); +// }); +// redisClient.quit(); +// }); // MySql Database // const connection = mysql.createConnection({ @@ -108,34 +130,48 @@ const _port = 1337; app.get('/*', (req, res) => { - console.log('Request coming in!', req.query) - + // console.log('Request coming in!', req.query) + const startTime = Date.now(); + statsDClient.increment('.service.house.query.all') let zipcode = req.query.zipcode || "94118"; let startDate = req.query.startDate.slice(0,4) + req.query.startDate.slice(5,7) || "200410"; let endDate = req.query.endDate.slice(0,4) + req.query.endDate.slice(5,7) || "200510"; - db.retrieveHomePrices("SELECT * FROM sf_home_sale_prices WHERE ZipCode=" + zipcode, function(data) { - if (data) { - var count = 0; - var responseObj = {}; - let sum = 0; - for (var key in data[0]) { - if (count > 0 || startDate === key) { - if (key === endDate) { - break; + + redisClient.get(JSON.stringify(req.query), function(err, reply) { + if (reply) { + res.json({average: reply}); + } else { + db.retrieveHomePrices("SELECT * FROM sf_home_sale_prices WHERE ZipCode=" + zipcode, function(data) { + if (data) { + var count = 0; + var responseObj = {}; + let sum = 0; + for (var key in data[0]) { + if (count > 0 || startDate === key) { + if (key === endDate) { + break; + } else { + count++; + sum += data[0][key]; + } + } + } + let average = sum/count; + if (average) { + res.json({average: average}) + statsDClient.timing('.service.house.query.latency_ms', Date.now() - startTime) + redisClient.set(JSON.stringify(req.query), 1200000, 'EX', 6); } else { - count++; - sum += data[0][key]; + res.json({average: '1200000'}) + statsDClient.timing('.service.house.query.latency_ms', Date.now() - startTime) + redisClient.set(JSON.stringify(req.query), 1200000, 'EX', 6); } + } else { + res.json({error: 'Please specify the ZipCode'}) + statsDClient.timing('.service.house.query.latency_ms', Date.now() - startTime) + statsDClient.increment('.service.house.query.fail') } - } - let average = sum/count; - if (average) { - res.json({average: average}) - } else { - res.json({average: 1200000}) - } - } else { - res.json({error: 'Please specify the ZipCode'}) + }) } }) });