I have node.js file that is connecting to the mongoDB and getting the number of the documents in the collection then it's running in for loop and updating each document with a value them I'm getting from an external server, the thing is that i have a lot of connections open at a time and the program is crashing, my question is how can i use 1 connection at a time and multiple connections this is my code:
var request = require('request');
var cheerio = require ('cheerio');
var fs = require('fs');
var MongoClient = require('mongodb').MongoClient;
//var config = JSON.parse(fs.readFileSync("config.json"));
//var host = config.host;
//var port = config.port;
// Variable Deceleration
//var dbName = "ystocks";
var port = "27017";
var host = "localhost";
var collection = "stocks"
var dbName = "yahooStocks";
function updateStock(){
// getting the number of docs in the collection
getNumOfDocs("stocks", host, port, dbName, function(err, count) {
if (err) {
return console.log(err.message);
}
console.log('number of documents', count);
for(i = 0 ; i <= 20; i++)
{
// Finding each document by _id with a callbak
GetDocumentByID("stocks", host, port, dbName, i, function(err, findOne) {
if (err) {
return console.log(err.message);
}
console.log('Here is the doc that was found', findOne.ticker);
var stockName = findOne.ticker;
console.log("\n* stock name : " +stockName + "\n");
var yUrl = "http://finance.yahoo.com/q/ks?s=" + stockName;
getStockValue(stockName, yUrl);
}); // end of GetDocumentByID
}//end of for loop
}); // end of getNumOfDocs call
}// end of update function
////passed arguments
//stockName = for printing stock name to console
//yUrl = the specific url for the current stock ticker
function getStockValue(stockName, yUrl){
request(yUrl, stockName, function (error, response, body) {
var date = setTimeToLocal(new Date(), 3);
if (!error && response.statusCode == 200) {
var $ = cheerio.load(body);
// the keys - We get them from a certain class attribute
var span = $('.time_rtq_ticker>span');
var stockValue = $(span).text();
console.log("checking the type of the stockValue " + typeof(stockValue));
//*************************************
// parsing the value to int in case it was a String
var parsedValue = parseFloat(stockValue);
console.log("checking the type of the stockValue " + typeof(parsedValue) + " " + parsedValue);
// Calling the setStockValue function which will update the stock value
setStockValue(stockName, parsedValue);
//writehead ???
//response.writeHead(200, {'Content-Type':'text/plain'});
console.log("Response received from -> " + yUrl);
console.log(date);
console.log("Stock - " + stockName + " --> " + stockValue );
//response.write is for printing in browser when we have a server running
//response.write("Welcome to stockwatch\n");
//response.write("Stock - " + ticker + " current value--> " + stockValue + "\n" );
//TODO #1
//ALEX : lines 61-72 -> should be replaced with a function call that
// connects to the DB and updates the specified stock
//i think it sould look like this : funcName( tickerName , currenValue ){...}
var json = {stockName : "", price : "", date : ""};
json.price = stockValue;
json.stockName = stockName;
json.date = date;
var tempName = json.stockName +'.json';
fs.writeFile( tempName ,JSON.stringify(json, null, 4) , function(err) {
if(!err) {
console.log("File successfully written");
}
});//end of filewrite
//end of TODO #1
//response.end("res end");
}//end of !error && response.statusCode == 200
else if (response.statusCode == 404){
console.log("Response failed from " + yUrl + " --> error code: " + response.statusCode);
}//end of statusCode == 400
});// end of request
}//end of getStockValue
// updating a stock with a provided ticker and value and the MongoClient
// edited by Alex Brodov on : AUG 21 2014
function setStockValue(ticker, value) {
// open the connection the DB server
MongoClient.connect("mongodb://localhost:27017/yahooStocks", function (error, db){
if(error) throw error;
var query = {'ticker' : ticker};
operator ={'$set' : {'value' : value}};
// find one document in our collection
db.collection('stocks').update(query, operator, function(error, updated){
if (error) throw error;
//print the result
console.dir("Successfully updated " + updated + " document");
// close the DB
db.close();
});// end of update
}); // Connection to the DB
} // end of setStockValue
// Gets the local date and the desired offset time
// set
function setTimeToLocal(date, offset ) {
// getting the local tome in millseconds
var localtime = date.getTime();
// getting the local offset in millseconds
var localOffset = date.getTimezoneOffset()*60000;
var utc = localOffset + localtime;
// Jerusalem offset
// var offset = 3;
// Jerusalem time in millseconds
var jerusalem = utc + (offset*3600000);
var d = new Date(jerusalem);
console.log("Jerusalem Local Time: " + d.toLocaleString());
return d;
} // end of SetTimeToLocal
/**
* Gets the number of documents from a collection
* With a provided collection name, DB host, DB port, DB name and a callback function
* Edited by Alex
*/
function getNumOfDocs (collectionName, host, port, dbName, callback) {
var mongoConnection = MongoClient.connect("mongodb://" + host + ":" + port + "/" + dbName, function (error, db){
if(error) return callback(err);
db.collection(collectionName).count({}, function(error, numOfDocs){
if(error) return callback(err);
db.close();
callback(null, numOfDocs);
}); // end of count
}); // end of connection to DB
} // end of getNumOfDocs
/**
* Gets a document
* With a provided collection name, DB host, DB port, DB name, the document ID and a callback
* Edited by Alex Brodov on Aug 24th 2014
*/
function GetDocumentByID (collectionName, host, port, dbName, docID, callback) {
// open the connection to DB server
MongoClient.connect("mongodb://" + host + ":" + port + "/" + dbName, function (error, db){
if(error) return callback(error);
var query = {'_id' : docID};
// find one document in our collection
db.collection(collectionName).findOne(query, function(error, doc) {
if (error) return callback(error);
console.log(doc);
// close the DB
db.close();
callback(null, doc);
});// end of findOne
}); // Connection to the DB
}
var refreshInterval = setInterval(updateStock, 30000);
