Getting Started with LocalStack and AWS S3Tables
export LOCALSTACK_AUTH_TOKEN="
mkdir -p .localstack-volume
docker rm -f localstack-main 2>/dev/null || true
docker run -d \
--name localstack-main \
-p 4566:4566 \
-p 4510-4559:4510-4559 \
-e LOCALSTACK_AUTH_TOKEN \
-e SERVICES=s3tables \
-v /var/run/docker.sock:/var/run/docker.sock \
-v "$(pwd)/.localstack-volume:/var/lib/localstack" \
localstack/localstack
In [32]:
! java --version
openjdk 11.0.25 2024-10-15
OpenJDK Runtime Environment Homebrew (build 11.0.25+0)
OpenJDK 64-Bit Server VM Homebrew (build 11.0.25+0, mixed mode)
#### Export ENV Variable
In [33]:
import os os.environ["AWS_ACCESS_KEY_ID"] = "test" os.environ["AWS_SECRET_ACCESS_KEY"] = "test" os.environ["AWS_DEFAULT_REGION"] = "us-east-1" os.environ['AWS_ENDPOINT_URL']="http://localhost.localstack.cloud:4566" os.environ["AWS_PROFILE"] = "localstack"
Create S3Tables
In [34]:
! aws s3tables list-table-buckets
{
"tableBuckets": []
}
In [35]:
! aws s3tables create-table-bucket --name my-table-bucket
{
"arn": "arn:aws:s3tables:us-east-1:000000000000:bucket/my-table-bucket"
}
🧪 Step 1 — Add Iceberg + AWS Packages
In [36]:
def iceberg_runtime_package():
import pyspark
major, minor = map(int, pyspark.__version__.split(".")[:2])
scala = "2.12" if major < 4 else "2.13"
iceberg_ver = "1.10.0"
return f"org.apache.iceberg:iceberg-spark-runtime-{major}.{minor}_{scala}:{iceberg_ver}"
ICEBERG_RUNTIME = iceberg_runtime_package()
packages = ",".join([
"com.amazonaws:aws-java-sdk-bundle:1.12.661",
"org.apache.hadoop:hadoop-aws:3.3.4",
"software.amazon.awssdk:bundle:2.29.38",
"com.github.ben-manes.caffeine:caffeine:3.1.8",
"org.apache.commons:commons-configuration2:2.11.0",
"software.amazon.s3tables:s3-tables-catalog-for-iceberg:0.1.8",
ICEBERG_RUNTIME,
])
os.environ["PYSPARK_SUBMIT_ARGS"] = f"--packages {packages} pyspark-shell"
print("Using Iceberg package:", ICEBERG_RUNTIME)Using Iceberg package: org.apache.iceberg:iceberg-spark-runtime-3.4_2.12:1.10.0
🧪 Step 2 — Create Spark Session
In [37]:
from pyspark.sql import SparkSession
import os
# Config (ensure these are already set)
TABLE_BUCKET_ARN = "arn:aws:s3tables:us-east-1:000000000000:bucket/my-table-bucket"
CATALOG_NAME = "ManagedIcebergCatalog"
spark = (
SparkSession.builder.appName("LocalStackS3TablesIceberg")
.config("spark.sql.extensions", "org.apache.iceberg.spark.extensions.IcebergSparkSessionExtensions")
.config(f"spark.sql.catalog.{CATALOG_NAME}", "org.apache.iceberg.spark.SparkCatalog")
.config(f"spark.sql.catalog.{CATALOG_NAME}.catalog-impl", "software.amazon.s3tables.iceberg.S3TablesCatalog")
.config(f"spark.sql.catalog.{CATALOG_NAME}.warehouse", TABLE_BUCKET_ARN)
.config(f"spark.sql.catalog.{CATALOG_NAME}.s3.endpoint", os.getenv('AWS_ENDPOINT_URL')) # <-- fixed
.config(f"spark.sql.catalog.{CATALOG_NAME}.s3.path-style-access", "true")
.config(f"spark.sql.catalog.{CATALOG_NAME}.s3.access-key-id", os.getenv("AWS_ACCESS_KEY_ID"))
.config(f"spark.sql.catalog.{CATALOG_NAME}.s3.secret-access-key", os.getenv("AWS_SECRET_ACCESS_KEY"))
.getOrCreate()
)
print("✅ Spark Session Created")✅ Spark Session Created
Perform Actions
In [38]:
cat = CATALOG_NAME
ns = "my_namespace"
table = "demo_table1"
spark.sql(f"CREATE NAMESPACE IF NOT EXISTS {CATALOG_NAME}.{ns}")
print("[ok] CREATE NAMESPACE IF NOT EXISTS", f"{CATALOG_NAME}.{ns}")[ok] CREATE NAMESPACE IF NOT EXISTS ManagedIcebergCatalog.my_namespace
In [39]:
spark.sql(
f"""
CREATE TABLE IF NOT EXISTS {CATALOG_NAME}.{ns}.{table} (
id STRING,
name STRING,
region STRING
)
USING iceberg
PARTITIONED BY (region)
TBLPROPERTIES ('format-version' = '2')
"""
).show()++
||
++
++
In [40]:
! aws s3tables list-table-buckets
{
"tableBuckets": [
{
"arn": "arn:aws:s3tables:us-east-1:000000000000:bucket/my-table-bucket",
"name": "my-table-bucket",
"ownerAccountId": "000000000000",
"createdAt": "2026-04-04T14:16:27.393500+00:00"
}
]
}
In [41]:
! aws s3tables list-tables --table-bucket-arn arn:aws:s3tables:us-east-1:000000000000:bucket/my-table-bucket
{
"tables": [
{
"namespace": [
"my_namespace"
],
"name": "demo_table1",
"type": "customer",
"tableARN": "arn:aws:s3tables:us-east-1:000000000000:bucket/my-table-bucket/table/demo_table1",
"createdAt": "2026-04-04T14:17:29.839974+00:00",
"modifiedAt": "2026-04-04T14:17:29.839974+00:00"
}
]
}
In [42]:
spark.sql(
f"""
INSERT INTO {cat}.{ns}.{table} (id, name, region) VALUES
('1', 'Alice', 'us-east-1'),
('2', 'Bob', 'eu-west-1')
"""
)
print("[ok] INSERT INTO ... (id, name, region) VALUES ...")
spark.sql(f"SELECT * FROM {cat}.{ns}.{table} ORDER BY id").show(truncate=False)
print("[ok] SELECT * ORDER BY id")[ok] INSERT INTO ... (id, name, region) VALUES ...
+---+-----+---------+
|id |name |region |
+---+-----+---------+
|1 |Alice|us-east-1|
|2 |Bob |eu-west-1|
+---+-----+---------+
[ok] SELECT * ORDER BY id
In [ ]:
