### Perform Kinesis Integration Test with Kinesalite Source: https://context7.com/mhart/kinesalite/llms.txt This example demonstrates how to programmatically start a Kinesalite server, configure the AWS SDK to interact with it, create a stream, perform record operations, and clean up resources. It uses the 'aws-sdk' and 'kinesalite' packages to simulate a full Kinesis lifecycle in a test environment. ```javascript var kinesalite = require('kinesalite') var AWS = require('aws-sdk') var server = kinesalite({ createStreamMs: 0, deleteStreamMs: 0, updateStreamMs: 0 }) var kinesis function setup(callback) { server.listen(0, function(err) { if (err) return callback(err) var port = server.address().port kinesis = new AWS.Kinesis({ endpoint: 'http://localhost:' + port, region: 'us-east-1', accessKeyId: 'test', secretAccessKey: 'test' }) callback() }) } function teardown(callback) { server.close(callback) } function runTest(callback) { var streamName = 'test-stream-' + Date.now() kinesis.createStream({ StreamName: streamName, ShardCount: 1 }, function(err) { if (err) return callback(err) kinesis.waitFor('streamExists', { StreamName: streamName }, function(err) { if (err) return callback(err) kinesis.putRecord({ StreamName: streamName, Data: JSON.stringify({ test: 'data' }), PartitionKey: 'key1' }, function(err, putResult) { if (err) return callback(err) kinesis.getShardIterator({ StreamName: streamName, ShardId: 'shardId-000000000000', ShardIteratorType: 'TRIM_HORIZON' }, function(err, iterResult) { if (err) return callback(err) kinesis.getRecords({ ShardIterator: iterResult.ShardIterator }, function(err, getResult) { if (err) return callback(err) if (getResult.Records.length !== 1) { return callback(new Error('Expected 1 record')) } var data = JSON.parse(getResult.Records[0].Data.toString()) if (data.test !== 'data') { return callback(new Error('Data mismatch')) } console.log('Test passed!') kinesis.deleteStream({ StreamName: streamName }, callback) }) }) }) }) }) } setup(function(err) { if (err) throw err runTest(function(err) { teardown(function() { if (err) throw err console.log('All tests completed successfully') }) }) }) ``` -------------------------------- ### Start Kinesalite Server (Programmatic) Source: https://context7.com/mhart/kinesalite/llms.txt Programmatically creates and starts a Kinesalite server instance using Node.js. Allows for detailed configuration of storage, SSL, stream state timings, and shard limits. The server can be started and later closed. ```javascript var kinesalite = require('kinesalite') var kinesaliteServer = kinesalite({ path: './mydb', ssl: false, createStreamMs: 50, deleteStreamMs: 50, updateStreamMs: 50, shardLimit: 10 }) kinesaliteServer.listen(4567, function(err) { if (err) throw err console.log('Kinesalite started on port 4567') }) kinesaliteServer.close(function(err) { if (err) throw err console.log('Kinesalite stopped') }) ``` -------------------------------- ### Start Kinesalite Server (CLI) Source: https://context7.com/mhart/kinesalite/llms.txt Starts the Kinesalite server using the command-line interface with various configuration options. Supports custom ports, persistent storage paths, SSL, and timing parameters for faster testing. It also allows setting a shard limit. ```bash kinesalite kinesalite --port 5000 --path ./kinesis-data kinesalite --port 4567 --ssl kinesalite --port 4567 --createStreamMs 50 --deleteStreamMs 50 --updateStreamMs 50 kinesalite --port 4567 --shardLimit 100 kinesalite --port 4567 --path ./mydb --ssl --createStreamMs 100 --deleteStreamMs 100 --updateStreamMs 100 --shardLimit 50 ``` -------------------------------- ### Connecting to Kinesalite with kinesis module (Node.js) Source: https://github.com/mhart/kinesalite/blob/master/README.md Illustrates how to connect to Kinesalite using the 'kinesis' Node.js module. This example is specifically noted to work with Kinesalite started in HTTPS mode. ```javascript var kinesis = require('kinesis') kinesis.listStreams({host: 'localhost', port: 4567}, console.log) ``` -------------------------------- ### Kinesalite Installation via npm Source: https://github.com/mhart/kinesalite/blob/master/README.md Provides the command to install Kinesalite globally using npm, making the 'kinesalite' command available in the system's PATH. ```sh npm install -g kinesalite ``` -------------------------------- ### Programmatic Kinesalite Server Initialization (Node.js) Source: https://github.com/mhart/kinesalite/blob/master/README.md Demonstrates how to programmatically start a Kinesalite server in Node.js. It shows how to require the 'kinesalite' module, configure it with options like 'path' and 'createStreamMs', and listen on a specified port. ```javascript var kinesalite = require('kinesalite'), kinesaliteServer = kinesalite({path: './mydb', createStreamMs: 50}) kinesaliteServer.listen(4567, function(err) { if (err) throw err console.log('Kinesalite started on port 4567') }) ``` -------------------------------- ### ListTagsForStream JavaScript Example Source: https://context7.com/mhart/kinesalite/llms.txt Lists all tags associated with a Kinesis stream and supports pagination. It fetches tags for a specified stream and handles cases where there are more tags than can be returned in a single request. ```javascript var AWS = require('aws-sdk') var kinesis = new AWS.Kinesis({ endpoint: 'http://localhost:4567' }) kinesis.listTagsForStream({ StreamName: 'my-test-stream', Limit: 10 }, function(err, data) { if (err) { console.error('Error:', err.message) return } console.log('Tags:') data.Tags.forEach(function(tag) { console.log(' ', tag.Key, '=', tag.Value) }) console.log('Has more tags:', data.HasMoreTags) // Paginate if needed if (data.HasMoreTags) { var lastKey = data.Tags[data.Tags.length - 1].Key kinesis.listTagsForStream({ StreamName: 'my-test-stream', ExclusiveStartTagKey: lastKey }, function(err, moreData) { if (err) console.error(err) else console.log('More tags:', moreData.Tags) }) } }) ``` -------------------------------- ### Connect to Kinesalite with AWS SDK Source: https://context7.com/mhart/kinesalite/llms.txt Configures the AWS SDK to connect to a local Kinesalite instance by specifying a custom endpoint. Dummy credentials can be used as Kinesalite does not verify signatures. This example demonstrates listing streams to test the connection. ```javascript var AWS = require('aws-sdk') AWS.config.update({ accessKeyId: 'fake', secretAccessKey: 'fake', region: 'us-east-1' }) var kinesis = new AWS.Kinesis({ endpoint: 'http://localhost:4567' }) kinesis.listStreams({}, function(err, data) { if (err) console.error(err) else console.log('Streams:', data.StreamNames) }) ``` -------------------------------- ### DecreaseStreamRetentionPeriod JavaScript Example Source: https://context7.com/mhart/kinesalite/llms.txt Decreases the data retention period for a Kinesis stream, with a minimum limit of 24 hours. It includes error handling for invalid arguments. ```javascript var AWS = require('aws-sdk') var kinesis = new AWS.Kinesis({ endpoint: 'http://localhost:4567' }) kinesis.decreaseStreamRetentionPeriod({ StreamName: 'my-test-stream', RetentionPeriodHours: 24 // Minimum retention period }, function(err, data) { if (err) { if (err.code === 'InvalidArgumentException') { console.log('Retention period must be >= 24 hours and < current period') } else { console.error('Error:', err.message) } return } console.log('Retention period decreased successfully') }) ``` -------------------------------- ### Manage Shard Iterators with GetShardIterator Source: https://context7.com/mhart/kinesalite/llms.txt Shows how to obtain a shard iterator for reading data from a Kinesis stream. Includes examples for various iterator types: TRIM_HORIZON, LATEST, specific sequence numbers, and timestamps. ```javascript var AWS = require('aws-sdk') var kinesis = new AWS.Kinesis({ endpoint: 'http://localhost:4567' }) kinesis.getShardIterator({ StreamName: 'my-test-stream', ShardId: 'shardId-000000000000', ShardIteratorType: 'TRIM_HORIZON' }, function(err, data) { if (err) { console.error('Error:', err.message) return } console.log('Iterator (TRIM_HORIZON):', data.ShardIterator.substring(0, 50) + '...') }) kinesis.getShardIterator({ StreamName: 'my-test-stream', ShardId: 'shardId-000000000000', ShardIteratorType: 'LATEST' }, function(err, data) { if (err) console.error(err) else console.log('Iterator (LATEST):', data.ShardIterator.substring(0, 50) + '...') }) kinesis.getShardIterator({ StreamName: 'my-test-stream', ShardId: 'shardId-000000000000', ShardIteratorType: 'AT_SEQUENCE_NUMBER', StartingSequenceNumber: '49590338271490256608559692538361571095921575989136588802' }, function(err, data) { if (err) console.error(err) else console.log('Iterator (AT_SEQUENCE_NUMBER):', data.ShardIterator.substring(0, 50) + '...') }) kinesis.getShardIterator({ StreamName: 'my-test-stream', ShardId: 'shardId-000000000000', ShardIteratorType: 'AFTER_SEQUENCE_NUMBER', StartingSequenceNumber: '49590338271490256608559692538361571095921575989136588802' }, function(err, data) { if (err) console.error(err) else console.log('Iterator (AFTER_SEQUENCE_NUMBER):', data.ShardIterator.substring(0, 50) + '...') }) kinesis.getShardIterator({ StreamName: 'my-test-stream', ShardId: 'shardId-000000000000', ShardIteratorType: 'AT_TIMESTAMP', Timestamp: new Date('2024-01-01T00:00:00Z') }, function(err, data) { if (err) console.error(err) else console.log('Iterator (AT_TIMESTAMP):', data.ShardIterator.substring(0, 50) + '...') }) ``` -------------------------------- ### IncreaseStreamRetentionPeriod JavaScript Example Source: https://context7.com/mhart/kinesalite/llms.txt Increases the data retention period for a Kinesis stream, with a maximum limit of 168 hours (7 days). It includes error handling for invalid arguments. ```javascript var AWS = require('aws-sdk') var kinesis = new AWS.Kinesis({ endpoint: 'http://localhost:4567' }) kinesis.increaseStreamRetentionPeriod({ StreamName: 'my-test-stream', RetentionPeriodHours: 168 // 7 days maximum }, function(err, data) { if (err) { if (err.code === 'InvalidArgumentException') { console.log('Retention period must be >= 24 hours and > current period') } else { console.error('Error:', err.message) } return } console.log('Retention period increased successfully') }) ``` -------------------------------- ### RemoveTagsFromStream JavaScript Example Source: https://context7.com/mhart/kinesalite/llms.txt Removes specified tags from a Kinesis stream. This function takes a stream name and an array of tag keys to be removed. ```javascript var AWS = require('aws-sdk') var kinesis = new AWS.Kinesis({ endpoint: 'http://localhost:4567' }) kinesis.removeTagsFromStream({ StreamName: 'my-test-stream', TagKeys: ['Environment', 'CostCenter'] }, function(err, data) { if (err) { console.error('Error:', err.message) return } console.log('Tags removed successfully') }) ``` -------------------------------- ### Connecting to Kinesalite with AWS SDK (Node.js) Source: https://github.com/mhart/kinesalite/blob/master/README.md Shows how to configure and use the AWS SDK for JavaScript to connect to a running Kinesalite instance. It demonstrates initializing the Kinesis client with a custom endpoint and performing a basic operation like listing streams. ```javascript var AWS = require('aws-sdk') var kinesis = new AWS.Kinesis({endpoint: 'http://localhost:4567'}) kinesis.listStreams(console.log.bind(console)) ``` -------------------------------- ### Kinesalite Command-Line Interface Help Source: https://github.com/mhart/kinesalite/blob/master/README.md Displays the help message for the Kinesalite command-line interface, outlining available options such as port, path, SSL, and various stream-related timings. ```sh kinesalite --help ``` -------------------------------- ### List Kinesis Streams with Pagination (JavaScript) Source: https://context7.com/mhart/kinesalite/llms.txt Lists all Kinesis streams in the account. Supports pagination to retrieve streams beyond the initial limit. Requires the AWS SDK. ```javascript var AWS = require('aws-sdk') var kinesis = new AWS.Kinesis({ endpoint: 'http://localhost:4567' }) // List all streams kinesis.listStreams({ Limit: 10 }, function(err, data) { if (err) { console.error('Error:', err.message) return } console.log('Streams:', data.StreamNames) console.log('Has more:', data.HasMoreStreams) // If there are more streams, paginate if (data.HasMoreStreams) { kinesis.listStreams({ Limit: 10, ExclusiveStartStreamName: data.StreamNames[data.StreamNames.length - 1] }, function(err, moreData) { if (err) console.error(err) else console.log('More streams:', moreData.StreamNames) }) } }) ``` -------------------------------- ### Batch Ingest Records with PutRecords Source: https://context7.com/mhart/kinesalite/llms.txt Demonstrates how to send multiple records to a Kinesis stream in a single request. It handles potential partial failures by checking the FailedRecordCount and iterating through individual record results. ```javascript var AWS = require('aws-sdk') var kinesis = new AWS.Kinesis({ endpoint: 'http://localhost:4567' }) var records = [ { Data: JSON.stringify({ event: 'page_view', page: '/home' }), PartitionKey: 'session-001' }, { Data: JSON.stringify({ event: 'page_view', page: '/products' }), PartitionKey: 'session-002' }, { Data: JSON.stringify({ event: 'add_to_cart', productId: 'ABC123' }), PartitionKey: 'session-001' }, { Data: JSON.stringify({ event: 'purchase', orderId: 'ORD-456' }), PartitionKey: 'session-003' } ] kinesis.putRecords({ StreamName: 'my-test-stream', Records: records }, function(err, data) { if (err) { console.error('Error:', err.message) return } console.log('Failed record count:', data.FailedRecordCount) data.Records.forEach(function(record, index) { if (record.ErrorCode) { console.log('Record', index, 'failed:', record.ErrorCode, record.ErrorMessage) } else { console.log('Record', index, 'written to', record.ShardId, 'seq:', record.SequenceNumber) } }) }) ``` -------------------------------- ### List Kinesis Shards with Filtering (JavaScript) Source: https://context7.com/mhart/kinesalite/llms.txt Lists all shards within a specified Kinesis stream. Supports filtering by 'ExclusiveStartShardId' for pagination. Requires the AWS SDK. ```javascript var AWS = require('aws-sdk') var kinesis = new AWS.Kinesis({ endpoint: 'http://localhost:4567' }) kinesis.listShards({ StreamName: 'my-test-stream' }, function(err, data) { if (err) { console.error('Error:', err.message) return } console.log('Shards:') data.Shards.forEach(function(shard) { console.log(' ID:', shard.ShardId) console.log(' Hash Range:', shard.HashKeyRange.StartingHashKey, '-', shard.HashKeyRange.EndingHashKey) }) }) // List shards starting from a specific shard ID kinesis.listShards({ StreamName: 'my-test-stream', ExclusiveStartShardId: 'shardId-000000000000' }, function(err, data) { if (err) console.error(err) else console.log('Shards after shard-0:', data.Shards.map(s => s.ShardId)) }) ``` -------------------------------- ### Describe Kinesis Stream Details (JavaScript) Source: https://context7.com/mhart/kinesalite/llms.txt Retrieves detailed information about a specific Kinesis stream, including its status, shards, and configuration. Handles 'ResourceNotFoundException'. Requires the AWS SDK. ```javascript var AWS = require('aws-sdk') var kinesis = new AWS.Kinesis({ endpoint: 'http://localhost:4567' }) kinesis.describeStream({ StreamName: 'my-test-stream' }, function(err, data) { if (err) { if (err.code === 'ResourceNotFoundException') { console.log('Stream not found') } else { console.error('Error:', err.message) } return } var stream = data.StreamDescription console.log('Stream Name:', stream.StreamName) console.log('Stream ARN:', stream.StreamARN) console.log('Stream Status:', stream.StreamStatus) console.log('Retention Period:', stream.RetentionPeriodHours, 'hours') console.log('Shard Count:', stream.Shards.length) stream.Shards.forEach(function(shard) { console.log(' Shard ID:', shard.ShardId) console.log(' Hash Range:', shard.HashKeyRange.StartingHashKey, '-', shard.HashKeyRange.EndingHashKey) console.log(' Starting Sequence:', shard.SequenceNumberRange.StartingSequenceNumber) }) }) ``` -------------------------------- ### Create Kinesis Stream with AWS SDK Source: https://context7.com/mhart/kinesalite/llms.txt Creates a new Kinesis stream using the AWS SDK connected to Kinesalite. Specifies the stream name and initial shard count. Includes error handling for common exceptions like 'ResourceInUseException' and 'LimitExceededException'. It also demonstrates waiting for the stream to become active. ```javascript var AWS = require('aws-sdk') var kinesis = new AWS.Kinesis({ endpoint: 'http://localhost:4567' }) kinesis.createStream({ StreamName: 'my-test-stream', ShardCount: 2 }, function(err, data) { if (err) { if (err.code === 'ResourceInUseException') { console.log('Stream already exists') } else if (err.code === 'LimitExceededException') { console.log('Shard limit exceeded') } else { console.error('Error:', err.message) } return } console.log('Stream creation initiated') kinesis.waitFor('streamExists', { StreamName: 'my-test-stream' }, function(err, data) { if (err) console.error(err) else console.log('Stream is now active:', data.StreamDescription.StreamStatus) }) }) ``` -------------------------------- ### AddTagsToStream API Source: https://context7.com/mhart/kinesalite/llms.txt Add metadata tags to a stream for organization and cost tracking. ```APIDOC ## AddTagsToStream ### Description Add metadata tags to a stream for organization and cost tracking. ### Method POST ### Endpoint `/` (Kinesis service endpoint) ### Parameters #### Path Parameters None #### Query Parameters None #### Request Body - **StreamName** (string) - Required - The name of the stream. - **Tags** (object) - Required - A map of tags to add to the stream. Keys and values must be strings. ### Request Example ```javascript var AWS = require('aws-sdk') var kinesis = new AWS.Kinesis({ endpoint: 'http://localhost:4567' }) kinesis.addTagsToStream({ StreamName: 'my-test-stream', Tags: { 'Environment': 'development', 'Application': 'my-app', 'CostCenter': 'engineering', 'Owner': 'data-team' } }, function(err, data) { if (err) { if (err.code === 'InvalidArgumentException') { console.log('Invalid tag characters or too many tags (max 50)') } else { console.error('Error:', err.message) } return } console.log('Tags added successfully') }) ``` ### Response #### Success Response (200) - **None** (object) - This operation does not return a response body on success. #### Response Example ```json {} ``` ``` -------------------------------- ### Retrieve Stream Records with GetRecords Source: https://context7.com/mhart/kinesalite/llms.txt Retrieves data records from a shard using an existing iterator. It handles the ExpiredIteratorException and demonstrates how to use the NextShardIterator for continuous polling. ```javascript var AWS = require('aws-sdk') var kinesis = new AWS.Kinesis({ endpoint: 'http://localhost:4567' }) kinesis.getShardIterator({ StreamName: 'my-test-stream', ShardId: 'shardId-000000000000', ShardIteratorType: 'TRIM_HORIZON' }, function(err, iteratorData) { if (err) { console.error('Error getting iterator:', err.message) return } kinesis.getRecords({ ShardIterator: iteratorData.ShardIterator, Limit: 100 }, function(err, data) { if (err) { if (err.code === 'ExpiredIteratorException') { console.log('Iterator expired, need to get a new one') } else { console.error('Error:', err.message) } return } console.log('Records retrieved:', data.Records.length) data.Records.forEach(function(record) { console.log(' Sequence:', record.SequenceNumber) console.log(' Data:', record.Data.toString()) }) if (data.NextShardIterator) { console.log('More records available, use NextShardIterator to continue') } else { console.log('Shard has been closed, no more records') } }) }) ``` -------------------------------- ### SplitShard API Source: https://context7.com/mhart/kinesalite/llms.txt Split a single shard into two shards to increase stream capacity. ```APIDOC ## SplitShard ### Description Split a single shard into two shards to increase stream capacity. ### Method POST ### Endpoint `/` (Kinesis service endpoint) ### Parameters #### Path Parameters None #### Query Parameters None #### Request Body - **StreamName** (string) - Required - The name of the stream. - **ShardToSplit** (string) - Required - The ID of the shard to split. - **NewStartingHashKey** (string) - Required - The hash key value for the starting shard of the two new shards. ### Request Example ```javascript var AWS = require('aws-sdk') var kinesis = new AWS.Kinesis({ endpoint: 'http://localhost:4567' }) // First describe the stream to get shard hash key ranges kinesis.describeStream({ StreamName: 'my-test-stream' }, function(err, data) { if (err) { console.error('Error:', err.message) return } var shard = data.StreamDescription.Shards[0] var startHash = BigInt(shard.HashKeyRange.StartingHashKey) var endHash = BigInt(shard.HashKeyRange.EndingHashKey) var midHash = ((startHash + endHash) / 2n).toString() console.log('Splitting shard:', shard.ShardId) console.log('New starting hash key:', midHash) // Split the shard at the midpoint kinesis.splitShard({ StreamName: 'my-test-stream', ShardToSplit: shard.ShardId, NewStartingHashKey: midHash }, function(err, data) { if (err) { if (err.code === 'ResourceInUseException') { console.log('Stream is not in ACTIVE state') } else if (err.code === 'LimitExceededException') { console.log('Would exceed shard limit') } else { console.error('Error:', err.message) } return } console.log('Shard split initiated, stream updating...') }) }) ``` ### Response #### Success Response (200) - **StreamStatus** (string) - The status of the stream (e.g., UPDATING). #### Response Example ```json { "StreamStatus": "UPDATING" } ``` ``` -------------------------------- ### Continuous Record Polling Source: https://context7.com/mhart/kinesalite/llms.txt Implement a continuous polling loop to consume records from a stream shard. ```APIDOC ## Continuous Record Polling ### Description Implement a continuous polling loop to consume records from a stream shard. ### Method GET (simulated via polling) ### Endpoint `/` (Kinesis service endpoint) ### Parameters #### Query Parameters - **streamName** (string) - Required - The name of the Kinesis stream. - **shardId** (string) - Required - The ID of the shard to poll. ### Request Example ```javascript var AWS = require('aws-sdk') var kinesis = new AWS.Kinesis({ endpoint: 'http://localhost:4567' }) function pollRecords(streamName, shardId) { // Get initial iterator kinesis.getShardIterator({ StreamName: streamName, ShardId: shardId, ShardIteratorType: 'LATEST' }, function(err, data) { if (err) { console.error('Error getting iterator:', err) return } var shardIterator = data.ShardIterator function poll() { if (!shardIterator) { console.log('Shard closed, stopping poll') return } kinesis.getRecords({ ShardIterator: shardIterator, Limit: 100 }, function(err, data) { if (err) { if (err.code === 'ExpiredIteratorException') { console.log('Iterator expired, reinitializing...') pollRecords(streamName, shardId) return } console.error('Error polling:', err) setTimeout(poll, 1000) return } // Process records data.Records.forEach(function(record) { var payload = JSON.parse(record.Data.toString()) console.log('Received:', payload) }) // Update iterator for next poll shardIterator = data.NextShardIterator // Poll again after a short delay setTimeout(poll, 1000) }) } poll() }) } // Start polling pollRecords('my-test-stream', 'shardId-000000000000') ``` ### Response #### Success Response (200) - **Records** (array) - A list of records retrieved from the shard. - **NextShardIterator** (string) - An iterator to use for the next poll request. #### Response Example ```json { "Records": [ { "SequenceNumber": "49590338271490257200912913144404349408000000000000000000", "ApproximateArrivalTimestamp": "2023-10-27T10:00:00.000Z", "Data": "{\"message\": \"Hello, Kinesis!\"}", "PartitionKey": "partitionKey-1" } ], "NextShardIterator": "shardIterator-abc123" } ``` ``` -------------------------------- ### Put Record into Kinesis Stream (JavaScript) Source: https://context7.com/mhart/kinesalite/llms.txt Writes a single data record to a specified Kinesis stream. Supports JSON or binary data and allows specifying a partition key or explicit hash key for shard targeting. Handles 'ResourceNotFoundException'. Requires the AWS SDK. ```javascript var AWS = require('aws-sdk') var kinesis = new AWS.Kinesis({ endpoint: 'http://localhost:4567' }) // Put a single record kinesis.putRecord({ StreamName: 'my-test-stream', Data: JSON.stringify({ event: 'user_login', userId: '12345', timestamp: Date.now() }), PartitionKey: 'user-12345' }, function(err, data) { if (err) { if (err.code === 'ResourceNotFoundException') { console.log('Stream not found') } else { console.error('Error:', err.message) } return } console.log('Record written to shard:', data.ShardId) console.log('Sequence number:', data.SequenceNumber) }) // Put record with explicit hash key for precise shard targeting kinesis.putRecord({ StreamName: 'my-test-stream', Data: Buffer.from('Binary data here'), PartitionKey: 'partition-1', ExplicitHashKey: '170141183460469231731687303715884105727' // Target specific shard }, function(err, data) { if (err) console.error(err) else console.log('Written to shard:', data.ShardId) }) ``` -------------------------------- ### Split Kinesis Shard (JavaScript) Source: https://context7.com/mhart/kinesalite/llms.txt Splits a single Kinesis stream shard into two to increase stream capacity. It first describes the stream to determine the hash key ranges and then initiates the split operation. Requires the AWS SDK. ```javascript var AWS = require('aws-sdk') var kinesis = new AWS.Kinesis({ endpoint: 'http://localhost:4567' }) // First describe the stream to get shard hash key ranges kinesis.describeStream({ StreamName: 'my-test-stream' }, function(err, data) { if (err) { console.error('Error:', err.message) return } var shard = data.StreamDescription.Shards[0] var startHash = BigInt(shard.HashKeyRange.StartingHashKey) var endHash = BigInt(shard.HashKeyRange.EndingHashKey) var midHash = ((startHash + endHash) / 2n).toString() console.log('Splitting shard:', shard.ShardId) console.log('New starting hash key:', midHash) // Split the shard at the midpoint kinesis.splitShard({ StreamName: 'my-test-stream', ShardToSplit: shard.ShardId, NewStartingHashKey: midHash }, function(err, data) { if (err) { if (err.code === 'ResourceInUseException') { console.log('Stream is not in ACTIVE state') } else if (err.code === 'LimitExceededException') { console.log('Would exceed shard limit') } else { console.error('Error:', err.message) } return } console.log('Shard split initiated, stream updating...') }) }) ``` -------------------------------- ### POST ListTagsForStream Source: https://context7.com/mhart/kinesalite/llms.txt Retrieves all tags associated with a specific Kinesis stream, supporting pagination. ```APIDOC ## POST ListTagsForStream ### Description Lists all tags associated with a stream. Supports pagination using ExclusiveStartTagKey. ### Method POST ### Endpoint / ### Parameters #### Request Body - **StreamName** (string) - Required - The name of the stream. - **Limit** (integer) - Optional - The maximum number of tags to return. - **ExclusiveStartTagKey** (string) - Optional - The key to start pagination from. ### Request Example { "StreamName": "my-test-stream", "Limit": 10 } ### Response #### Success Response (200) - **Tags** (array) - List of tag objects containing Key and Value. - **HasMoreTags** (boolean) - Indicates if more tags are available. #### Response Example { "Tags": [{"Key": "Environment", "Value": "Production"}], "HasMoreTags": false } ``` -------------------------------- ### POST IncreaseStreamRetentionPeriod Source: https://context7.com/mhart/kinesalite/llms.txt Increases the data retention period for a stream up to 168 hours. ```APIDOC ## POST IncreaseStreamRetentionPeriod ### Description Increases the data retention period for a stream. The value must be greater than the current period and up to a maximum of 168 hours. ### Method POST ### Endpoint / ### Parameters #### Request Body - **StreamName** (string) - Required - The name of the stream. - **RetentionPeriodHours** (integer) - Required - The new retention period in hours. ### Request Example { "StreamName": "my-test-stream", "RetentionPeriodHours": 168 } ``` -------------------------------- ### POST DecreaseStreamRetentionPeriod Source: https://context7.com/mhart/kinesalite/llms.txt Decreases the data retention period for a stream, with a minimum of 24 hours. ```APIDOC ## POST DecreaseStreamRetentionPeriod ### Description Decreases the data retention period for a stream. The value must be less than the current period and at least 24 hours. ### Method POST ### Endpoint / ### Parameters #### Request Body - **StreamName** (string) - Required - The name of the stream. - **RetentionPeriodHours** (integer) - Required - The new retention period in hours. ### Request Example { "StreamName": "my-test-stream", "RetentionPeriodHours": 24 } ``` -------------------------------- ### Add Tags to Kinesis Stream (JavaScript) Source: https://context7.com/mhart/kinesalite/llms.txt Adds metadata tags to a Kinesis stream for organization and cost tracking. This function takes a stream name and a map of tags as input. It utilizes the AWS SDK for the operation. ```javascript var AWS = require('aws-sdk') var kinesis = new AWS.Kinesis({ endpoint: 'http://localhost:4567' }) kinesis.addTagsToStream({ StreamName: 'my-test-stream', Tags: { 'Environment': 'development', 'Application': 'my-app', 'CostCenter': 'engineering', 'Owner': 'data-team' } }, function(err, data) { if (err) { if (err.code === 'InvalidArgumentException') { console.log('Invalid tag characters or too many tags (max 50)') } else { console.error('Error:', err.message) } return } console.log('Tags added successfully') }) ``` -------------------------------- ### Merge Kinesis Shards (JavaScript) Source: https://context7.com/mhart/kinesalite/llms.txt Merges two adjacent Kinesis stream shards into one to reduce stream capacity. It describes the stream to find open shards and then initiates the merge operation. Requires the AWS SDK. ```javascript var AWS = require('aws-sdk') var kinesis = new AWS.Kinesis({ endpoint: 'http://localhost:4567' }) // First describe stream to find adjacent shards kinesis.describeStream({ StreamName: 'my-test-stream' }, function(err, data) { if (err) { console.error('Error:', err.message) return } var shards = data.StreamDescription.Shards.filter(function(shard) { return !shard.SequenceNumberRange.EndingSequenceNumber // Only open shards }) if (shards.length < 2) { console.log('Need at least 2 open shards to merge') return } // Find adjacent shards (where one's ending hash + 1 = other's starting hash) var shard1 = shards[0] var shard2 = shards[1] console.log('Merging shards:', shard1.ShardId, 'and', shard2.ShardId) kinesis.mergeShards({ StreamName: 'my-test-stream', ShardToMerge: shard1.ShardId, AdjacentShardToMerge: shard2.ShardId }, function(err, data) { if (err) { if (err.code === 'InvalidArgumentException') { console.log('Shards are not adjacent') } else if (err.code === 'ResourceInUseException') { console.log('Stream is not in ACTIVE state') } else { console.error('Error:', err.message) } return } console.log('Shard merge initiated, stream updating...') }) }) ``` -------------------------------- ### MergeShards API Source: https://context7.com/mhart/kinesalite/llms.txt Merge two adjacent shards into one to reduce stream capacity. ```APIDOC ## MergeShards ### Description Merge two adjacent shards into one to reduce stream capacity. ### Method POST ### Endpoint `/` (Kinesis service endpoint) ### Parameters #### Path Parameters None #### Query Parameters None #### Request Body - **StreamName** (string) - Required - The name of the stream. - **ShardToMerge** (string) - Required - The ID of the first shard to merge. - **AdjacentShardToMerge** (string) - Required - The ID of the second, adjacent shard to merge. ### Request Example ```javascript var AWS = require('aws-sdk') var kinesis = new AWS.Kinesis({ endpoint: 'http://localhost:4567' }) // First describe stream to find adjacent shards kinesis.describeStream({ StreamName: 'my-test-stream' }, function(err, data) { if (err) { console.error('Error:', err.message) return } var shards = data.StreamDescription.Shards.filter(function(shard) { return !shard.SequenceNumberRange.EndingSequenceNumber // Only open shards }) if (shards.length < 2) { console.log('Need at least 2 open shards to merge') return } // Find adjacent shards (where one's ending hash + 1 = other's starting hash) var shard1 = shards[0] var shard2 = shards[1] console.log('Merging shards:', shard1.ShardId, 'and', shard2.ShardId) kinesis.mergeShards({ StreamName: 'my-test-stream', ShardToMerge: shard1.ShardId, AdjacentShardToMerge: shard2.ShardId }, function(err, data) { if (err) { if (err.code === 'InvalidArgumentException') { console.log('Shards are not adjacent') } else if (err.code === 'ResourceInUseException') { console.log('Stream is not in ACTIVE state') } else { console.error('Error:', err.message) } return } console.log('Shard merge initiated, stream updating...') }) }) ``` ### Response #### Success Response (200) - **StreamStatus** (string) - The status of the stream (e.g., UPDATING). #### Response Example ```json { "StreamStatus": "UPDATING" } ``` ``` -------------------------------- ### Poll Kinesis Stream Records Continuously (JavaScript) Source: https://context7.com/mhart/kinesalite/llms.txt Implements a continuous polling loop to consume records from a Kinesis stream shard. It handles iterator expiration and processes records by parsing their data payload. Dependencies include the AWS SDK. ```javascript var AWS = require('aws-sdk') var kinesis = new AWS.Kinesis({ endpoint: 'http://localhost:4567' }) function pollRecords(streamName, shardId) { // Get initial iterator kinesis.getShardIterator({ StreamName: streamName, ShardId: shardId, ShardIteratorType: 'LATEST' }, function(err, data) { if (err) { console.error('Error getting iterator:', err) return } var shardIterator = data.ShardIterator function poll() { if (!shardIterator) { console.log('Shard closed, stopping poll') return } kinesis.getRecords({ ShardIterator: shardIterator, Limit: 100 }, function(err, data) { if (err) { if (err.code === 'ExpiredIteratorException') { console.log('Iterator expired, reinitializing...') pollRecords(streamName, shardId) return } console.error('Error polling:', err) setTimeout(poll, 1000) return } // Process records data.Records.forEach(function(record) { var payload = JSON.parse(record.Data.toString()) console.log('Received:', payload) }) // Update iterator for next poll shardIterator = data.NextShardIterator // Poll again after a short delay setTimeout(poll, 1000) }) } poll() }) } // Start polling pollRecords('my-test-stream', 'shardId-000000000000') ``` -------------------------------- ### Delete Kinesis Stream with AWS SDK Source: https://context7.com/mhart/kinesalite/llms.txt Deletes an existing Kinesis stream using the AWS SDK connected to Kinesalite. Handles the 'ResourceNotFoundException' if the stream does not exist. The stream enters a DELETING state before removal. ```javascript var AWS = require('aws-sdk') var kinesis = new AWS.Kinesis({ endpoint: 'http://localhost:4567' }) kinesis.deleteStream({ StreamName: 'my-test-stream' }, function(err, data) { if (err) { if (err.code === 'ResourceNotFoundException') { console.log('Stream does not exist') } else { console.error('Error:', err.message) } return } console.log('Stream deletion initiated') }) ``` -------------------------------- ### POST RemoveTagsFromStream Source: https://context7.com/mhart/kinesalite/llms.txt Removes specified tags from a Kinesis stream. ```APIDOC ## POST RemoveTagsFromStream ### Description Removes the specified tags from the given Kinesis stream. ### Method POST ### Endpoint / ### Parameters #### Request Body - **StreamName** (string) - Required - The name of the stream. - **TagKeys** (array) - Required - A list of tag keys to remove. ### Request Example { "StreamName": "my-test-stream", "TagKeys": ["Environment", "CostCenter"] } ### Response #### Success Response (200) - **Status** (string) - Confirmation of successful removal. ``` === COMPLETE CONTENT === This response contains all available snippets from this library. No additional content exists. Do not make further requests.