Skip to content
Snippets Groups Projects

Draft: Add Spark Structured Streaming based Benchmarks

Open Christian Richter requested to merge stu204011/theodolite:theodolite-spark into main
5 unresolved threads
3 files
+ 71
0
Compare changes
  • Side-by-side
  • Inline
Files
3
apiVersion: apps/v1
kind: Deployment
metadata:
name: spark-driver
labels:
app: spark
spec:
selector:
matchLabels:
app: spark
component: driver
replicas: 1
template:
metadata:
labels:
app: spark
component: driver
spec:
containers:
- name: spark
image: "happyente/uc1-spark:latest"
args: [
"driver"
, "--master", "spark://spark-master:7077"
# , "--executor-memory" , "4G" # If resource nodes lacks resources (for example if limited by kubernetes), this will result in failure!!!
# , "--total-executor-cores", "1"
, "--class" , "org.apache.spark.examples.JavaSparkPi"
, "--conf" , "spark.driver.port=7077"
, "--conf" , "spark.driver.blockManager.port=7078"
, "--conf" , "spark.driver.host=spark-driver" # TODO what is the right value here???
, "--conf" , "spark.cores.max=1"
, "/opt/spark/examples/jars/spark-examples_2.12-3.2.0.jar"
]
env:
- name: SPARK_DRIVER_BIND_ADDRESS
value: "0.0.0.0"
ports:
- containerPort: 7077
name: driver-rpc # TODO what kind of interface is deployed?
- containerPort: 7078
name: blockmanager
Loading