Implement JavaScript Buffer to store data

Question

I receive these huge Strings via WebSocket:

[
  "BTC-31DEC21-100000-P",
  "{\"data\":{\"bids\":{\"0.01\":{\"price\":0.01,\"volume\":66.2,\"exchange\":\"DER\"},\"5.0E-4\":{\"price\":5.0E-4,\"volume\":1.1,\"exchange\":\"DER\"},\"0.637\":{\"price\":0.637,\"volume\":8.4,\"exchange\":\"DER\"}},\"asks\":{\"0.664\":{\"price\":0.664,\"volume\":8.4,\"exchange\":\"DER\"}}},\"isMasterFrame\":true}"
]

or

[
  "BTC-31DEC21-36000-C",
  "{\"data\":[{\"price\":0.422,\"volume\":8.4,\"exchange\":\"DER\",\"side\":\"ASKS\"},{\"price\":0.423,\"volume\":0.0,\"exchange\":\"DER\",\"side\":\"ASKS\"}],\"isMasterFrame\":false}"
]

or

[
  "BTC-31DEC21-60000-P",
  "{\"data\":[{\"price\":0.105,\"volume\":0.0,\"exchange\":\"DER\",\"side\":\"ASKS\"},{\"price\":0.1055,\"volume\":28.7,\"exchange\":\"DER\",\"side\":\"ASKS\"},{\"price\":0.106,\"volume\":7.6,\"exchange\":\"DER\",\"side\":\"ASKS\"},{\"price\":0.1065,\"volume\":43.0,\"exchange\":\"DER\",\"side\":\"ASKS\"}],\"isMasterFrame\":false}"
]

I want to make JavaScript Buffer where I can store the data and reset it if I receive JSON with isMasterFrame = true

let payload = JSON.parse(messageString[1]);

if (payload.hasOwnProperty("isMasterFrame")) {
  for (let i = 0; i < payload.pairs.length; i++) {
    let currentPair = payload.data[i]
    currentPair = currentPair.replace(/\0/g, ''); //Remove null chars
    if (currentPair.toUpperCase() != 'KILL') {
      // reset the buffer and start again to fill data
    }
  }
} else {
  // if we receive "isMasterFrame":false just update the data without reset
}

What are the available options to implement this buffer?

Why not if(payload.data.isMasterFrame) { ... }, and where is the pairs property in the data (did I miss it)? It also appears the reset is conditional upon the currentPair not being equal to KILL. Is that true? — Randy Casburn
– Randy Casburn, Commented Nov 18, 2021 at 23:39
other than those clarifcations, it is as simple as creating a cache array (let pairsCache = [];, then .push() onto the array if not a masterframe and not KILL otherwise set the cache array back to pairsCache = []; — Randy Casburn
– Randy Casburn, Commented Nov 18, 2021 at 23:43
you're right. This is is just and example from other code. Can you show me code example please for the solution? — user1285928
– user1285928, Commented Nov 19, 2021 at 7:41
Can you please provided a data sample that contains "KILL" ? — Randy Casburn
– Randy Casburn, Commented Nov 19, 2021 at 18:40

Randy Casburn · Accepted Answer · 2021-11-20 01:27:30Z

2

I think this is what you are after. It caches the details until it finds isMasterFrame: true && KILL, at which time it clears the cache and starts over again.

EDIT: Stored data using BTC style string as key and data as value to enable querying cache.

let messageCache = {};
const messages = [
  ["BTC-31DEC21-36000-C", "{\"data\":[{\"price\":0.422,\"volume\":8.4,\"exchange\":\"DER\",\"side\":\"ASKS\"},{\"price\":0.423,\"volume\":0.0,\"exchange\":\"DER\",\"side\":\"ASKS\"}],\"isMasterFrame\":false}"],
  ["BTC-31DEC21-36000-D", "{\"data\":[{\"price\":0.422,\"volume\":8.4,\"exchange\":\"DER\",\"side\":\"ASKS\"},{\"price\":0.423,\"volume\":0.0,\"exchange\":\"DER\",\"side\":\"ASKS\"}],\"isMasterFrame\":false}"],
  ["BTC-31DEC21-60000-P", "{\"data\":[{\"price\":0.105,\"volume\":0.0,\"exchange\":\"DER\",\"side\":\"ASKS\"},{\"price\":0.1055,\"volume\":28.7,\"exchange\":\"DER\",\"side\":\"ASKS\"},{\"price\":0.106,\"volume\":7.6,\"exchange\":\"DER\",\"side\":\"ASKS\"},{\"price\":0.1065,\"volume\":43.0,\"exchange\":\"DER\",\"side\":\"KILL\"}],\"isMasterFrame\"\:true}"],
  ["BTC-31DEC21-60000-Q", "{\"data\":[{\"price\":0.105,\"volume\":0.0,\"exchange\":\"DER\",\"side\":\"ASKS\"},{\"price\":0.1055,\"volume\":28.7,\"exchange\":\"DER\",\"side\":\"ASKS\"},{\"price\":0.106,\"volume\":7.6,\"exchange\":\"DER\",\"side\":\"ASKS\"},{\"price\":0.1065,\"volume\":43.0,\"exchange\":\"DER\",\"side\":\"ASKS\"}],\"isMasterFrame\"\:false}"]
];

function processData(key, payload) {
  if (!payload.isMasterFrame) {
    messageCache[key] = payload; 
    return;
  }
  for (let obj of payload.data) {
    Object.values(obj).forEach(item => {
      item = item.toString().replace(/\0/g, ''); //Remove null chars
      if (item.toUpperCase() !== 'KILL') {
        messageCache = {}; // Clear the cache/buffer
      }
    });
  }
}

function queryCache(key){
   return messageCache[key];
}

messages.forEach(message => {
  const payload = JSON.parse(message[1]);
  processData(message[0], payload);
  console.log("Number of cached messages: " + Object.keys(messageCache).length);
});

console.log('Query Cache [BTC-31DEC21-60000-Q]:');
let result = queryCache('BTC-31DEC21-60000-Q');
console.log(result);

edited Nov 20, 2021 at 1:27

answered Nov 19, 2021 at 20:41

Randy Casburn

14.2k1 gold badge19 silver badges32 bronze badges

Sign up to request clarification or add additional context in comments.

5 Comments

user1285928 Over a year ago

I will check it. By the way one additional question. I expect this code to consume too much memory. Is it possible to use more efficient memory structure? Also can you show me how I can query the data?

user1285928 Over a year ago

For example for BTC-31DEC21-36000-C I want to get all available data from the buffer.

Randy Casburn Over a year ago

Please address the memory consumption in a different question SO. It is frowned upon to ask multiple questions in one. I recommend you discover if you have a memory issue prior to assuming you will.

Randy Casburn Over a year ago

In order to query on that string, that code needs to change. You were not clear about what was to go into the "buffer", so I didn't included that string. The code needs to change, I'll get to it soon.

Randy Casburn Over a year ago

@user1285928 - Answer has been edited to allow querying. Please accept ✓ this answer if you find it helpful. Thanks.

Collectives™ on Stack Overflow

Implement JavaScript Buffer to store data

1 Answer 1

5 Comments

Your Answer

Hot Network Questions

Collectives™ on Stack Overflow

1 Answer 1

5 Comments

Your Answer

Sign up or log in

Post as a guest

Related