AWS Lambda - Sample Functions

Fivetran users who signed up on or after July 22, 2025, no longer have access to Function connections. Older Fivetran users can continue to create and use Function connections as before. All our users can use Connector SDK to build custom connectors. Connector SDK provides a streamlined development experience, and we'll host your custom connector for you.

Code samples to help you build your Lambda Function. The pieces of code show what an AWS Lambda function implementation looks like in different programming languages.

We have included sample functions for you to use to accelerate your function development:

A sample function request and the generated response to help you understand Fivetran's request and response format and how to use the different nodes to write your cloud function.
A transport API sample function to retrieve data from the Transport API.
A marketstack API sample function to retrieve data from the MarketStack API.

Sample function request

Sync directly

Select your language to view the sample function request:

package main

import (
	"github.com/aws/aws-lambda-go/lambda"
)

type Request struct {
	State   map[string]string `json:"state"`
	Secrets map[string]string `json:"secrets"`
}

type Response struct {
	State   map[string]string   `json:"state"`
	Insert  map[string][]Record `json:"insert"`
	Delete  map[string][]Record `json:"delete"`
	Schema  map[string]Key      `json:"schema"`
	HasMore bool                `json:"hasMore"`
}

type Key struct {
	PrimaryKey []string `json:"primary_key"`
}

type Record struct {
	Date     string `json:"date"`
	OrderId  int    `json:"order_id"`
	Amount   string `json:"amount"`
	Discount string `json:"discount"`
}

func handleLambdaEvent(request Request) (Response, error) {
	insertTransactions, deleteTransactions, newTransactionsCursor := apiResponse(request.State, request.Secrets)
	newState := make(map[string]string)
	newState["transactionsCursor"] = newTransactionsCursor
	primary_key := make([]string, 0)
	primary_key = append(primary_key, "order_id")
	primary_key = append(primary_key, "date")
	transactionsSchema := make(map[string]Key)
	transactionsSchema["transactions"] = Key{PrimaryKey: primary_key}
	return Response{State: newState, Insert: insertTransactions, Delete: deleteTransactions, Schema: transactionsSchema, HasMore: false}, nil
}

func apiResponse(state map[string]string, secrets map[string]string) (map[string][]Record, map[string][]Record, string) {
	insertTransactions := make(map[string][]Record)
	insertTransactions["transactions"] = append(insertTransactions["transactions"], Record{Date: "2017-12-31T05:12:05Z", OrderId: 1001, Amount: "$1200", Discount: "$12"})
	insertTransactions["transactions"] = append(insertTransactions["transactions"], Record{Date: "2017-12-31T05:12:05Z", OrderId: 1002, Amount: "$1345", Discount: "$14"})
	deleteTransactions := make(map[string][]Record)
	deleteTransactions["transactions"] = append(deleteTransactions["transactions"], Record{Date: "2017-12-31T05:12:05Z", OrderId: 1001, Amount: "$1200", Discount: "$12"})
	return insertTransactions, deleteTransactions, "2018-01-01T00:00:00Z"
}

func main() {
	lambda.Start(handleLambdaEvent)
}

Java

// Main.java
package com.fivetran.function;

import java.time.Instant;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import com.amazonaws.services.lambda.runtime.Context;
import com.amazonaws.services.lambda.runtime.RequestHandler;

public class Main implements RequestHandler {
    @Override
    public Response handleRequest(Request request, Context context) {
        return apiResponse(request);
    }

    Response apiResponse(Request request) {
        Record record1 = new Record(Instant.parse("2017-12-31T05:12:05Z"), 1001L, "$1200", "$12");
        Record record2 = new Record(Instant.parse("2017-12-31T06:12:04Z"), 1001L, "$1200", "$12");
        Record record3 = new Record(Instant.parse("2017-12-31T05:12:05Z"), 1000L, "$1200", "$12");
        Record record4 = new Record(Instant.parse("2017-12-31T06:12:04Z"), 1000L, "$1200", "$12");
        HashMap newState = new HashMap<>();
        newState.put("transactionCursor", Instant.parse("2018-01-01T00:00:00Z"));
        request.state = newState;

        //insert records
        List records = new ArrayList<>();
        records.add(record1);
        records.add(record2);
        Map insert = new HashMap<>();
        insert.put("transactions", records);

        //delete records
        records = new ArrayList<>();
        records.add(record3);
        records.add(record4);
        Map delete = new HashMap<>();
        delete.put("transactions", records);

        // schema
        List primaryKey = new ArrayList<>();
        primaryKey.add("order_id");
        primaryKey.add("date");
        Map> transactionsSchema = new HashMap<>();
        transactionsSchema.put("primary_key", primaryKey);
        Map schema = new HashMap<>();
        schema.put("transactions", transactionsSchema);

        return new Response(request.state, insert, delete, schema, true);
    }
}

// Record.java
package com.fivetran.function;

import java.time.Instant;
public class Record {
    public Instant date;
    public Long order_id;
    public String amount;
    public String discount;
    
    public Record(Instant date, Long order_id, String amount, String discount) {
        this.date = date;
        this.order_id = order_id;
        this.amount = amount;
        this.discount = discount;
    }
}

// Request.java
package com.fivetran.function;

public class Request {
    public Object state;
    public Object secrets;
}

// Response.java
package com.fivetran.function;
import java.util.Map;
public class Response {
    public Object state;
    public Map insert;
    public Map delete;
    public Map schema;
    public boolean hasMore;
    
    public Response(
            Object state,
            Map insert,
            Map delete,
            Map schema,
            boolean hasMore) {
        this.state = state;
        this.insert = insert;
        this.delete = delete;
        this.schema = schema;
        this.hasMore = hasMore;
    }
}

// pom.xml

  4.0.0
  com.fivetran.function
  sample
  jar
  1.0
  sample
  http://maven.apache.org

  
    11
    11
  
  
    
      
        org.apache.maven.plugins
        maven-shade-plugin
        3.2.2
        
          false
        
        
          
            package
            
              shade
            
          
        
      
      
        org.apache.maven.plugins
        maven-compiler-plugin
        
          8
          8
        
      
    
  
  
    
      com.amazonaws
      aws-lambda-java-core
      1.2.1
    
    
      com.amazonaws
      aws-lambda-java-events
      3.11.0
    
    
      com.amazonaws
      aws-lambda-java-log4j2
      1.5.1
    
    
      com.fasterxml.jackson.core
      jackson-databind
      2.13.1

Node.js

exports.handler = (request, context, callback) => {
    callback(null, update(request.state, request.secrets));
};
function update(state, secrets) {
    // Fetch records using api calls
    let [insertTransactions, deleteTransactions, newTransactionsCursor] = apiResponse(state, secrets);
    // Populate records and state
    return ({
        state: {
            transactionsCursor: newTransactionsCursor
        },
        insert: {
            transactions: insertTransactions
        },
        delete: {
            transactions: deleteTransactions
        },
        schema : {
            transactions : {
                primary_key : ['order_id', 'date']
            }
        },
        hasMore : false
    });
}
function apiResponse(state, secrets) {
    var insertTransactions = [
            {"date":'2017-12-31T05:12:05Z', "order_id":1001, "amount":'$1200', "discount":'$12'},
            {"date":'2017-12-31T06:12:04Z', "order_id":1001, "amount":'$1200', "discount":'$12'},
    ];
    var deleteTransactions = [
            {"date":'2017-12-31T05:12:05Z', "order_id":1000, "amount":'$1200', "discount":'$12'},
            {"date":'2017-12-31T06:12:04Z', "order_id":1000, "amount":'$1200', "discount":'$12'},
    ];
    return [insertTransactions, deleteTransactions, '2018-01-01T00:00:00Z'];
}

Python

import json
def lambda_handler(request, context):
    # Fetch records using api calls
    (insertTransactions, deleteTransactions, newTransactionCursor) = api_response(request['state'], request['secrets'])
    # Populate records in insert
    insert = {}
    insert['transactions'] = insertTransactions
    delete = {}
    delete['transactions'] = deleteTransactions
    state = {}
    state['transactionsCursor'] = newTransactionCursor
    transactionsSchema = {}
    transactionsSchema['primary_key'] = ['order_id', 'date']
    schema = {}
    schema['transactions'] = transactionsSchema
    response = {}
    # Add updated state to response
    response['state'] =  state
    # Add all the records to be inserted in response
    response['insert'] = insert
    # Add all the records to be marked as deleted in response
    response['delete'] = delete
    # Add schema defintion in response
    response['schema'] = schema
    # Add hasMore flag
    response['hasMore'] = False
    return response
def api_response(state, secrets):
    # your api call goes here
    insertTransactions = [
            {"date":'2017-12-31T05:12:05Z', "order_id":1001, "amount":'$1200', "discount":'$12'},
            {"date":'2017-12-31T06:12:04Z', "order_id":1001, "amount":'$1200', "discount":'$12'},
    ]
    deleteTransactions = [
            {"date":'2017-12-31T05:12:05Z', "order_id":1000, "amount":'$1200', "discount":'$12'},
            {"date":'2017-12-31T06:12:04Z', "order_id":1000, "amount":'$1200', "discount":'$12'},
    ]
    newTransactionCursor='2018-01-01T00:00:00Z'
    return (insertTransactions, deleteTransactions, newTransactionCursor)

For more information about the fields in the request, see our request format documentation.

Generated response

The sample function request generates the following JSON response:

{
  "state": {
    "transactionsCursor": "2018-01-01T00:00:00Z"
  },
  "insert": {
    "transactions": [
      {
        "date": "2017-12-31T05:12:05Z",
        "order_id": 1001,
        "amount": "$1200",
        "discount": "$12"
      },
      {
        "date": "2017-12-31T06:12:04Z",
        "order_id": 1001,
        "amount": "$1200",
        "discount": "$12"
      }
    ]
  },
  "delete": {
    "transactions": [
      {
        "date": "2017-12-31T05:12:05Z",
        "order_id": 1000,
        "amount": "$1200",
        "discount": "$12"
      },
      {
        "date": "2017-12-31T06:12:04Z",
        "order_id": 1000,
        "amount": "$1200",
        "discount": "$12"
      }
    ]
  },
  "schema": {
    "transactions": {
      "primary_key": [
        "order_id",
        "date"
      ]
    }
  },
  "hasMore": "true"
}

For more information about the fields in the response, see our response format documentation.

Sync through S3 bucket

Select your language to view the sample function request:

package main

import (
	"bytes"
	"context"
	"encoding/json"
	"github.com/aws/aws-lambda-go/lambda"
	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/s3/s3manager"
	"os"
)

type Request struct {
	State   map[string]string `json:"state"`
	Secrets map[string]string `json:"secrets"`
	Bucket  string            `json:"bucket"`
	File    string            `json:"file"`
}

type Response struct {
	State   map[string]string `json:"state"`
	Schema  map[string]Key    `json:"schema"`
	HasMore bool              `json:"hasMore"`
}

type S3Response struct {
	Insert map[string][]Record `json:"insert"`
	Delete map[string][]Record `json:"delete"`
}

type Key struct {
	PrimaryKey []string `json:"primary_key"`
}

type Record struct {
	Date     string `json:"date"`
	OrderId  int    `json:"order_id"`
	Amount   string `json:"amount"`
	Discount string `json:"discount"`
}

func handleLambdaEvent(request Request) (Response, error) {
	insertTransactions, deleteTransactions, newTransactionsCursor := apiResponse(request.State, request.Secrets)
	newState := make(map[string]string)
	newState["transactionsCursor"] = newTransactionsCursor
	primary_key := make([]string, 0)
	primary_key = append(primary_key, "order_id")
	primary_key = append(primary_key, "date")
	transactionsSchema := make(map[string]Key)
	transactionsSchema["transactions"] = Key{PrimaryKey: primary_key}

	json := generateJson(insertTransactions, deleteTransactions)
	uploadToS3(request.Bucket, request.File, json)

	return Response{State: newState, Schema: transactionsSchema, HasMore: false}, nil
}

func apiResponse(state map[string]string, secrets map[string]string) (map[string][]Record, map[string][]Record, string) {
	insertTransactions := make(map[string][]Record)
	insertTransactions["transactions"] = append(insertTransactions["transactions"], Record{Date: "2017-12-31T05:12:05Z", OrderId: 1001, Amount: "$1200", Discount: "$12"})
	insertTransactions["transactions"] = append(insertTransactions["transactions"], Record{Date: "2017-12-31T05:12:05Z", OrderId: 1002, Amount: "$1345", Discount: "$14"})
	deleteTransactions := make(map[string][]Record)
	deleteTransactions["transactions"] = append(deleteTransactions["transactions"], Record{Date: "2017-12-31T05:12:05Z", OrderId: 1001, Amount: "$1200", Discount: "$12"})
	return insertTransactions, deleteTransactions, "2018-01-01T00:00:00Z"
}

func generateJson(insert map[string][]Record, delete map[string][]Record) []byte {
	response := &S3Response{Insert: insert, Delete: delete}
	json, err := json.Marshal(response)

	if err != nil {
		panic(err)
	}

	return json
}

func uploadToS3(bucket string, file string, json []byte) {
	// create session and uploader
	s3Config := &aws.Config{
		//we need to specify the bucket region here if bucket is not global & `lambda` in not the same region as s3 bucket
		Region:      aws.String(os.Getenv("AWS_REGION")), 
	}
	s3Session := session.New(s3Config)
	uploader := s3manager.NewUploader(s3Session)

	// uploading
	input := &s3manager.UploadInput{
		Bucket:      aws.String(bucket),
		Key:         aws.String(file),
		Body:        bytes.NewReader(json),
		ContentType: aws.String("json"),
	}
	_, err := uploader.UploadWithContext(context.Background(), input)

	if err != nil {
		panic(err)
	}
}

func main() {
	lambda.Start(handleLambdaEvent)
}

Java

// Main.java
package com.fivetran.function;

import com.amazonaws.SdkClientException;
import com.amazonaws.regions.Regions;
import com.amazonaws.services.lambda.runtime.Context;
import com.amazonaws.services.lambda.runtime.RequestHandler;
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.AmazonS3ClientBuilder;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;

import java.time.Instant;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

public class Main implements RequestHandler {
    private static final ObjectMapper MAPPER = new ObjectMapper().registerModule(new JavaTimeModule());

    public Response handleRequest(final Request request, final Context context) {
        return this.apiResponse(request);
    }

    Response apiResponse(Request request) {
        Record record1 = new Record(Instant.parse("2017-12-31T05:12:05Z"), 1001L, "$1200", "$12");
        Record record2 = new Record(Instant.parse("2017-12-31T06:12:04Z"), 1001L, "$1200", "$12");
        Record record3 = new Record(Instant.parse("2017-12-31T05:12:05Z"), 1000L, "$1200", "$12");
        Record record4 = new Record(Instant.parse("2017-12-31T06:12:04Z"), 1000L, "$1200", "$12");
        HashMap newState = new HashMap<>();
        newState.put("transactionCursor", Instant.parse("2018-01-01T00:00:00Z"));
        request.state = newState;

        //insert records
        List records = new ArrayList<>();
        records.add(record1);
        records.add(record2);
        Map insert = new HashMap<>();
        insert.put("transactions", records);

        //delete records
        records = new ArrayList<>();
        records.add(record3);
        records.add(record4);
        Map delete = new HashMap<>();
        delete.put("transactions", records);

        // schema
        List primaryKey = new ArrayList<>();
        primaryKey.add("order_id");
        primaryKey.add("date");
        Map> transactionsSchema = new HashMap<>();
        transactionsSchema.put("primary_key", primaryKey);
        Map schema = new HashMap<>();
        schema.put("transactions", transactionsSchema);

        this.pushToS3(request.bucket, request.file, this.jsonString(insert, delete));
        return new Response(request.state, schema, true);
    }

    private String jsonString(final Map insert, final Map delete) {
        try {
            return Main.MAPPER.writeValueAsString(Map.of("insert", insert, "delete", delete));
        } catch (JsonProcessingException e) {
            throw new RuntimeException("Failed while generating json", e);
        }
    }

    private void pushToS3(final String bucket, final String objKey, final String json) {
        try {
            //we need to specify the bucket region here if bucket is not global & `lambda` in not the same region as s3 bucket
            AmazonS3 s3Client = AmazonS3ClientBuilder.standard().withRegion(Regions.AP_SOUTH_1).build();
            s3Client.putObject(bucket, objKey, json);
        } catch (SdkClientException e) {
            throw new RuntimeException(String.format("Failed while pushing %s object to %s bucket", objKey, bucket), e);
        }
    }
}

// Record.java
package com.fivetran.function;

import java.time.Instant;

public class Record {
    public Instant date;
    public Long order_id;
    public String amount;
    public String discount;

    public Record(Instant date, Long order_id, String amount, String discount) {
        this.date = date;
        this.order_id = order_id;
        this.amount = amount;
        this.discount = discount;
    }
}

// Request.java
package com.fivetran.function;

public class Request {
    public Object state;
    public Object secrets;
    public String bucket;
    public String file;
}

// Response.java
package com.fivetran.function;

import java.util.Map;

public class Response {
    public Object state;
    public Map schema;
    public boolean hasMore;

    public Response(
            Object state,
            Map schema,
            boolean hasMore) {
        this.state = state;
        this.schema = schema;
        this.hasMore = hasMore;
    }
}

// pom.xml

  4.0.0
  com.fivetran.function
  sample
  jar
  1.0
  sample
  http://maven.apache.org

  
    11
    11
  
  
    
      
        org.apache.maven.plugins
        maven-shade-plugin
        3.2.2
        
          false
        
        
          
            package
            
              shade
            
          
        
      
      
        org.apache.maven.plugins
        maven-compiler-plugin
        
          9
          9
        
      
    
  
  
    
      com.amazonaws
      aws-lambda-java-core
      1.2.1
    
    
      com.amazonaws
      aws-lambda-java-events
      3.11.0
    
    
      com.amazonaws
      aws-lambda-java-log4j2
      1.5.1
    
    
      com.fasterxml.jackson.core
      jackson-databind
      2.13.3
    
    
      com.fasterxml.jackson.datatype
      jackson-datatype-jsr310
      2.13.3
    
    
      com.amazonaws
      aws-java-sdk-s3
      1.12.264

Node.js

exports.handler = (request, context, callback) => {
    callback(null, update(request.state, request.secrets,request.bucket,request.file));
};
function update(state, secrets,bucket,key) {
    // Fetch records using api calls
    let [insertTransactions, deleteTransactions, newTransactionsCursor] = apiResponse(state, secrets);

    // Populate insert and delete in records
    var records=JSON.stringify({
        insert: {
            transactions: insertTransactions
        },
        delete: {
            transactions: deleteTransactions
        }
    })

    // Store records in s3 bucket
    putObjectToS3(bucket,key,records)

    // Return response
    return ({
        state: {
            transactionsCursor: newTransactionsCursor
        },
        schema : {
            transactions : {
                primary_key : ['order_id', 'date']
            }
        },
        hasMore : false
    });
}
function apiResponse(state, secrets) {
    var insertTransactions = [
            {"date":'2017-12-31T05:12:05Z', "order_id":1001, "amount":'$1200', "discount":'$12'},
            {"date":'2017-12-31T06:12:04Z', "order_id":1001, "amount":'$1200', "discount":'$12'},
    ];
    var deleteTransactions = [
            {"date":'2017-12-31T05:12:05Z', "order_id":1000, "amount":'$1200', "discount":'$12'},
            {"date":'2017-12-31T06:12:04Z', "order_id":1000, "amount":'$1200', "discount":'$12'},
    ];
    return [insertTransactions, deleteTransactions, '2018-01-01T00:00:00Z'];
}

// Function to store data in s3 bucket
var AWS = require('aws-sdk');
function putObjectToS3(bucket, key, data){
    var s3 = new AWS.S3();
        var params = {
            Bucket : bucket,
            Key : key,
            Body : data
        }
        s3.putObject(params, function(err, data) {
          if (err) console.log(err, err.stack); // an error occurred
          else     console.log(data);           // successful response
        });
}

Python

import json
import boto3

def lambda_handler(request, context):
    # Fetch records using api calls
    (insertTransactions, deleteTransactions, newTransactionCursor) = api_response(request['state'], request['secrets'])
    # Populate records in insert
    insert = {}
    insert['transactions'] = insertTransactions
    delete = {}
    delete['transactions'] = deleteTransactions
    state = {}
    state['transactionsCursor'] = newTransactionCursor
    transactionsSchema = {}
    transactionsSchema['primary_key'] = ['order_id', 'date']
    schema = {}
    schema['transactions'] = transactionsSchema
    response = {}
    # Add updated state to response
    response['state'] =  state
    # Add all the records to be inserted in response
    response['schema'] = schema
    # Add hasMore flag
    response['hasMore'] = False

    response_s3={}
    # Add all the records to be inserted in response_s3
    response_s3['insert'] = insert
    # Add all the records to be marked as deleted in response_s3
    response_s3['delete'] = delete

    json_str = bytes(json.dumps(response_s3).encode("utf-8"))

    # Push data in s3
    push_data_s3(request, json_str)

    return response

def api_response(state, secrets):
    # your api call goes here
    insertTransactions = [
            {"date":'2017-12-31T05:12:05Z', "order_id":1001, "amount":'$1200', "discount":'$12'},
            {"date":'2017-12-31T06:12:04Z', "order_id":1001, "amount":'$1200', "discount":'$12'},
    ]
    deleteTransactions = [
            {"date":'2017-12-31T05:12:05Z', "order_id":1000, "amount":'$1200', "discount":'$12'},
            {"date":'2017-12-31T06:12:04Z', "order_id":1000, "amount":'$1200', "discount":'$12'},
    ]
    newTransactionCursor='2018-01-01T00:00:00Z'
    return (insertTransactions, deleteTransactions, newTransactionCursor)

#Function to store data in s3
def push_data_s3(request, json_str):
    client = boto3.client('s3')
    client.put_object(Body=json_str, Bucket=request["bucket"], Key=request["file"])

For more information about the fields in the request, see our request format documentation. You can access all the files in our GitHub repository.

Generated response

Function Response

The sample function request returned the following JSON response:

{
  "state": {
    "transactionsCursor": "2018-01-01T00:00:00Z"
  },
  "schema": {
    "transactions": {
      "primary_key": [
        "order_id",
        "date"
      ]
    }
  },
  "hasMore": "true"
}

Response in S3 Bucket

The sample function pushed the following JSON response in the specified S3 file:

{
  "insert": {
    "transactions": [
      {
        "date": "2017-12-31T05:12:05Z",
        "order_id": 1001,
        "amount": "$1200",
        "discount": "$12"
      },
      {
        "date": "2017-12-31T06:12:04Z",
        "order_id": 1001,
        "amount": "$1200",
        "discount": "$12"
      }
    ]
  },
  "delete": {
    "transactions": [
      {
        "date": "2017-12-31T05:12:05Z",
        "order_id": 1000,
        "amount": "$1200",
        "discount": "$12"
      },
      {
        "date": "2017-12-31T06:12:04Z",
        "order_id": 1000,
        "amount": "$1200",
        "discount": "$12"
      }
    ]
  }
}

For more information about the fields in the response, see our response format documentation.

Transport API sample function

Download our sample Transport - API (London Tube) function. Select your language to download the sample function (.zip file):

Upload the downloaded .zip file to the Lambda console. For more information, see Lambda deployment packages.

Marketstack API sample function

See our sample Marketstack - API (Ticker) Python function.

From the main.py file, create a .zip file and then upload it to the Lambda console. For more information, see Lambda deployment packages. For more information about the Marketstack API, see our README.