JavaScript イテレータヘルパーバッチングエンジン：スケーラブルなアプリケーションのためのバッチ処理の最適化

            
function* numberGenerator(max) {
  let i = 0;
  while (i < max) {
    yield i++;
  }
}

const iterator = numberGenerator(5);
console.log(iterator.next()); // Output: { value: 0, done: false }
console.log(iterator.next()); // Output: { value: 1, done: false }
console.log(iterator.next()); // Output: { value: 2, done: false }
console.log(iterator.next()); // Output: { value: 3, done: false }
console.log(iterator.next()); // Output: { value: 4, done: false }
console.log(iterator.next()); // Output: { value: undefined, done: true }

            

              
                Copy
              
            
          

            
async function* asyncNumberGenerator(max) {
  let i = 0;
  while (i < max) {
    await new Promise(resolve => setTimeout(resolve, 100)); // Simulate async operation
    yield i++;
  }
}

async function consumeAsyncIterator() {
  const iterator = asyncNumberGenerator(5);
  let result = await iterator.next();
  while (!result.done) {
    console.log(result.value);
    result = await iterator.next();
  }
}

consumeAsyncIterator();

            

              
                Copy
              
            
          

            
function batchIterator(iterator, batchSize) {
  return {
    next() {
      const batch = [];
      for (let i = 0; i < batchSize; i++) {
        const result = iterator.next();
        if (result.done) {
          if (batch.length > 0) {
            return { value: batch, done: false };
          } else {
            return { value: undefined, done: true };
          }
        }
        batch.push(result.value);
      }
      return { value: batch, done: false };
    }
  };
}

// Example usage:
const numbers = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10];
const numberIterator = numbers[Symbol.iterator]();
const batchedIterator = batchIterator(numberIterator, 3);

let batchResult = batchedIterator.next();
while (!batchResult.done) {
  console.log('Batch:', batchResult.value);
  batchResult = batchedIterator.next();
}

            

              
                Copy
              
            
          

            
async function* asyncBatchIterator(asyncIterator, batchSize) {
  let batch = [];
  for await (const item of asyncIterator) {
    batch.push(item);
    if (batch.length === batchSize) {
      yield batch;
      batch = [];
    }
  }
  if (batch.length > 0) {
    yield batch;
  }
}

// Example Usage:
async function* generateAsyncNumbers(max) {
  for (let i = 0; i < max; i++) {
    await new Promise(resolve => setTimeout(resolve, 50)); // Simulate async operation
    yield i;
  }
}

async function processBatches() {
  const asyncNumberGeneratorInstance = generateAsyncNumbers(15);
  const batchedAsyncIterator = asyncBatchIterator(asyncNumberGeneratorInstance, 4);

  for await (const batch of batchedAsyncIterator) {
    console.log('Async Batch:', batch);
  }
}

processBatches();

            

              
                Copy
              
            
          

            
async function processBatchesConcurrently(asyncIterator, batchSize, concurrency) {
  const batchedAsyncIterator = asyncBatchIterator(asyncIterator, batchSize);
  const workers = Array(concurrency).fill(null).map(async () => {
    for await (const batch of batchedAsyncIterator) {
      // Process the batch concurrently
      await processBatch(batch);
    }
  });
  await Promise.all(workers);
}

async function processBatch(batch) {
  // Simulate batch processing
  await new Promise(resolve => setTimeout(resolve, 200));
  console.log('Processed batch:', batch);
}

            

              
                Copy
              
            
          

            
async function processBatchWithRetry(batch, maxRetries = 3) {
  let retries = 0;
  while (retries < maxRetries) {
    try {
      await processBatch(batch);
      return;
    } catch (error) {
      console.error(`Error processing batch (retry ${retries + 1}):`, error);
      retries++;
      await new Promise(resolve => setTimeout(resolve, 1000)); // Wait before retrying
    }
  }
  console.error('Failed to process batch after multiple retries:', batch);
}

            

              
                Copy
              
            
          

            
async function insertRecordsInBatches(records, batchSize, db) {
  const recordIterator = records[Symbol.iterator]();
  const batchedRecordIterator = batchIterator({
      next: () => {
          const next = recordIterator.next();
          return {value: next.value, done: next.done};
      }
  }, batchSize);

  let batchResult = batchedRecordIterator.next();
  while (!batchResult.done) {
    const batch = batchResult.value;
    try {
      await db.insertMany(batch);
      console.log(`Inserted batch of ${batch.length} records.`);
    } catch (error) {
      console.error('Error inserting batch:', error);
    }
    batchResult = batchedRecordIterator.next();
  }
  console.log('Finished inserting all records.');
}

// Example usage (assuming a MongoDB connection):
async function main() {
  const { MongoClient } = require('mongodb');
  const uri = 'mongodb://localhost:27017';
  const client = new MongoClient(uri);

  try {
    await client.connect();
    const db = client.db('mydb');
    const collection = db.collection('mycollection');

    const records = Array(1000).fill(null).map((_, i) => ({
      id: i + 1,
      name: `Record ${i + 1}`,
      timestamp: new Date()
    }));

    await insertRecordsInBatches(records, 100, collection);
  } catch (e) {
    console.error(e);
  } finally {
    await client.close();
  }
}

main();

            

              
                Copy