Skip to content

Commit fe3dbeb

Browse files
bpkrothbpkroth
authored
Enable chbenchmark in most CI pipelines (#519)
SqlServer support will happen in #517 - [x] oracle chbenchmark - ~~cockroachdb chbenchmark~~ skipping for now - see #525
1 parent dbf3e89 commit fe3dbeb

File tree

7 files changed

+449
-17
lines changed

7 files changed

+449
-17
lines changed

‎.github/workflows/maven.yml

+29-5
Original file line numberDiff line numberDiff line change
@@ -115,7 +115,7 @@ jobs:
115115
fail-fast: false
116116
matrix:
117117
# BROKEN: tpch
118-
benchmark: [ 'epinions', 'hyadapt', 'noop', 'otmetrics', 'resourcestresser', 'seats', 'sibench', 'smallbank', 'tatp', 'templated', 'tpcc', 'twitter', 'voter', 'wikipedia', 'ycsb' ]
118+
benchmark: [ 'chbenchmark', 'epinions', 'hyadapt', 'noop', 'otmetrics', 'resourcestresser', 'seats', 'sibench', 'smallbank', 'tatp', 'templated', 'tpcc', 'twitter', 'voter', 'wikipedia', 'ycsb' ]
119119
steps:
120120
- name: Download artifact
121121
uses: actions/download-artifact@v4
@@ -149,6 +149,10 @@ jobs:
149149
echo "The ${{matrix.benchmark}} benchmark is not supported for sqlite."
150150
exit 0
151151
else
152+
if [ ${{matrix.benchmark}} == chbenchmark ]; then
153+
# Disable synchronous mode for sqlite tpcc data loading to save some time.
154+
java -jar benchbase.jar -b tpcc -c config/sqlite/sample_tpcc_nosync_config.xml --create=true --load=true --execute=false --json-histograms results/histograms.json
155+
fi
152156
java -jar benchbase.jar -b ${{matrix.benchmark}} -c config/sqlite/sample_${{matrix.benchmark}}_config.xml --create=true --load=true --execute=true --json-histograms results/histograms.json
153157
fi
154158
@@ -179,7 +183,7 @@ jobs:
179183
fail-fast: false
180184
matrix:
181185
# FIXME: Add tpch back in (#333).
182-
benchmark: [ 'auctionmark', 'epinions', 'hyadapt', 'noop', 'otmetrics', 'resourcestresser', 'seats', 'sibench', 'smallbank', 'tatp', 'templated', 'tpcc', 'tpcc-with-reconnects', 'twitter', 'voter', 'wikipedia', 'ycsb' ]
186+
benchmark: [ 'auctionmark', 'chbenchmark', 'epinions', 'hyadapt', 'noop', 'otmetrics', 'resourcestresser', 'seats', 'sibench', 'smallbank', 'tatp', 'templated', 'tpcc', 'tpcc-with-reconnects', 'twitter', 'voter', 'wikipedia', 'ycsb' ]
183187
services:
184188
mariadb: # https://hub.docker.com/_/mariadb
185189
image: mariadb:latest
@@ -233,6 +237,9 @@ jobs:
233237
(sleep 10 && ./scripts/interrupt-docker-db-service.sh mariadb) &
234238
java -jar benchbase.jar -b tpcc -c config/mariadb/sample_tpcc_config.xml --execute=true --json-histograms results/histograms.json
235239
else
240+
if [ ${{matrix.benchmark}} == chbenchmark ]; then
241+
java -jar benchbase.jar -b tpcc -c config/mariadb/sample_tpcc_config.xml --create=true --load=true --execute=false --json-histograms results/histograms.json
242+
fi
236243
java -jar benchbase.jar -b ${{matrix.benchmark}} -c config/mariadb/sample_${{matrix.benchmark}}_config.xml --create=true --load=true --execute=true --json-histograms results/histograms.json
237244
fi
238245
@@ -258,7 +265,7 @@ jobs:
258265
strategy:
259266
fail-fast: false
260267
matrix:
261-
benchmark: [ 'auctionmark', 'epinions', 'hyadapt', 'noop', 'otmetrics', 'resourcestresser', 'seats', 'sibench', 'smallbank', 'tatp', 'templated', 'tpcc', 'tpcc-with-reconnects', 'tpch', 'twitter', 'voter', 'wikipedia', 'ycsb' ]
268+
benchmark: [ 'auctionmark', 'chbenchmark', 'epinions', 'hyadapt', 'noop', 'otmetrics', 'resourcestresser', 'seats', 'sibench', 'smallbank', 'tatp', 'templated', 'tpcc', 'tpcc-with-reconnects', 'tpch', 'twitter', 'voter', 'wikipedia', 'ycsb' ]
262269
services:
263270
mysql: # https://hub.docker.com/_/mysql
264271
image: mysql:latest
@@ -311,6 +318,9 @@ jobs:
311318
(sleep 10 && ./scripts/interrupt-docker-db-service.sh mysql) &
312319
java -jar benchbase.jar -b tpcc -c config/mysql/sample_tpcc_config.xml --execute=true --json-histograms results/histograms.json
313320
else
321+
if [ ${{matrix.benchmark}} == chbenchmark ]; then
322+
java -jar benchbase.jar -b tpcc -c config/mysql/sample_tpcc_config.xml --create=true --load=true --execute=false --json-histograms results/histograms.json
323+
fi
314324
java -jar benchbase.jar -b ${{matrix.benchmark}} -c config/mysql/sample_${{matrix.benchmark}}_config.xml --create=true --load=true --execute=true --json-histograms results/histograms.json
315325
fi
316326
@@ -336,7 +346,7 @@ jobs:
336346
strategy:
337347
fail-fast: false
338348
matrix:
339-
benchmark: [ 'auctionmark', 'epinions', 'hyadapt', 'noop', 'otmetrics', 'resourcestresser', 'seats', 'sibench', 'smallbank', 'tatp', 'templated', 'tpcc', 'tpcc-with-reconnects', 'tpch', 'twitter', 'voter', 'wikipedia', 'ycsb' ]
349+
benchmark: [ 'auctionmark', 'chbenchmark', 'epinions', 'hyadapt', 'noop', 'otmetrics', 'resourcestresser', 'seats', 'sibench', 'smallbank', 'tatp', 'templated', 'tpcc', 'tpcc-with-reconnects', 'tpch', 'twitter', 'voter', 'wikipedia', 'ycsb' ]
340350
services:
341351
oracle:
342352
image: gvenzl/oracle-xe:21.3.0-slim-faststart
@@ -393,6 +403,9 @@ jobs:
393403
(sleep 10 && ./scripts/interrupt-docker-db-service.sh oracle) &
394404
java -jar benchbase.jar -b tpcc -c config/oracle/sample_tpcc_config.xml --execute=true --json-histograms results/histograms.json
395405
else
406+
if [ ${{matrix.benchmark}} == chbenchmark ]; then
407+
java -jar benchbase.jar -b tpcc -c config/oracle/sample_tpcc_config.xml --create=true --load=true --execute=false --json-histograms results/histograms.json
408+
fi
396409
java -jar benchbase.jar -b ${{matrix.benchmark}} -c config/oracle/sample_${{matrix.benchmark}}_config.xml --create=true --load=true --execute=true --json-histograms results/histograms.json
397410
fi
398411
@@ -424,7 +437,7 @@ jobs:
424437
strategy:
425438
fail-fast: false
426439
matrix:
427-
benchmark: [ 'auctionmark', 'epinions', 'hyadapt', 'noop', 'otmetrics', 'resourcestresser', 'seats', 'sibench', 'smallbank', 'tatp', 'templated', 'tpcc', 'tpcc-with-reconnects', 'tpch', 'twitter', 'voter', 'wikipedia', 'ycsb' ]
440+
benchmark: [ 'auctionmark', 'chbenchmark', 'epinions', 'hyadapt', 'noop', 'otmetrics', 'resourcestresser', 'seats', 'sibench', 'smallbank', 'tatp', 'templated', 'tpcc', 'tpcc-with-reconnects', 'tpch', 'twitter', 'voter', 'wikipedia', 'ycsb' ]
428441
steps:
429442
# Note: we download just the docker-compose scripts/configs rather than the
430443
# whole source code repo for better testing.
@@ -479,6 +492,9 @@ jobs:
479492
(sleep 10 && ./scripts/interrupt-docker-db-service.sh postgres) &
480493
java -jar benchbase.jar -b tpcc -c config/postgres/sample_tpcc_config.xml -im 1000 -mt advanced --execute=true --json-histograms results/histograms.json
481494
else
495+
if [ ${{matrix.benchmark}} == chbenchmark ]; then
496+
java -jar benchbase.jar -b tpcc -c config/postgres/sample_tpcc_config.xml --create=true --load=true --execute=false --json-histograms results/histograms.json
497+
fi
482498
java -jar benchbase.jar -b ${{matrix.benchmark}} -c config/postgres/sample_${{matrix.benchmark}}_config.xml -im 1000 -mt advanced --create=true --load=true --execute=true --json-histograms results/histograms.json
483499
fi
484500
@@ -515,12 +531,14 @@ jobs:
515531
fail-fast: false
516532
matrix:
517533
# TODO: Add tpcc-with-reconnects benchmark support
534+
# TODO: Add chbenchmark benchmark support
518535
benchmark: [ 'auctionmark', 'epinions', 'hyadapt', 'noop', 'otmetrics', 'resourcestresser', 'seats', 'sibench', 'smallbank', 'tatp', 'templated', 'tpcc', 'tpch', 'twitter', 'voter', 'wikipedia', 'ycsb' ]
519536
services:
520537
cockroach: # https://hub.docker.com/repository/docker/timveil/cockroachdb-single-node
521538
image: timveil/cockroachdb-single-node:latest
522539
env:
523540
DATABASE_NAME: benchbase
541+
# TODO: Expand for additional config adjustments (See Also: #405, #519, #525)
524542
MEMORY_SIZE: .75
525543
ports:
526544
- 26257:26257
@@ -557,6 +575,9 @@ jobs:
557575
(sleep 10 && ./scripts/interrupt-docker-db-service.sh cockroachdb) &
558576
java -jar benchbase.jar -b tpcc -c config/cockroachdb/sample_tpcc_config.xml --execute=true --json-histograms results/histograms.json
559577
else
578+
if [ ${{matrix.benchmark}} == chbenchmark ]; then
579+
java -jar benchbase.jar -b tpcc -c config/cockroachdb/sample_tpcc_config.xml --create=true --load=true --execute=false --json-histograms results/histograms.json
580+
fi
560581
java -jar benchbase.jar -b ${{matrix.benchmark}} -c config/cockroachdb/sample_${{matrix.benchmark}}_config.xml --create=true --load=true --execute=true --json-histograms results/histograms.json
561582
fi
562583
@@ -655,6 +676,9 @@ jobs:
655676
(sleep 10 && ./scripts/interrupt-docker-db-service.sh sqlserver) &
656677
java -jar benchbase.jar -b tpcc -c config/sqlserver/sample_tpcc_config.xml -im 1000 -mt advanced --execute=true --json-histograms results/histograms.json
657678
else
679+
if [ ${{matrix.benchmark}} == chbenchmark ]; then
680+
java -jar benchbase.jar -b tpcc -c config/sqlserver/sample_tpcc_config.xml --create=true --load=true --execute=false --json-histograms results/histograms.json
681+
fi
658682
java -jar benchbase.jar -b ${{matrix.benchmark}} -c config/sqlserver/sample_${{matrix.benchmark}}_config.xml -im 1000 -mt advanced --create=true --load=true --execute=true --json-histograms results/histograms.json
659683
fi
660684

‎config/sqlite/sample_chbenchmark_config.xml

+161
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,161 @@
1+
<?xml version="1.0"?>
2+
<parameters>
3+
4+
<!-- Connection details -->
5+
<type>SQLITE</type>
6+
<driver>org.sqlite.JDBC</driver>
7+
<url>jdbc:sqlite:tpcc.db</url>
8+
<isolation>TRANSACTION_SERIALIZABLE</isolation>
9+
<batchsize>128</batchsize>
10+
11+
<!-- Scale factor is the number of warehouses in TPCC -->
12+
<scalefactor>1</scalefactor>
13+
14+
<!-- SQLITE only supports one writer thread -->
15+
<loaderThreads>1</loaderThreads>
16+
17+
<!-- The workload -->
18+
<!-- Number of terminal per workload -->
19+
<terminals>1</terminals>
20+
21+
<!-- Extra Features (Commented Out) -->
22+
<!-- Can be workload-specific -->
23+
<!-- <terminals bench="tpcc">2</terminals> -->
24+
25+
<!-- Workload-specific options a marked with @bench=[workload_name] -->
26+
<!-- Workload-specific number of terminals -->
27+
<!-- <terminals bench="chbenchmark">2</terminals> -->
28+
29+
<works>
30+
31+
<!-- A Basic WorkPhase for Mixed Workloads -->
32+
<work>
33+
<time>60</time>
34+
35+
<!-- Note: The rate can be set to UNLIMITED or DISABLED -->
36+
<rate>200</rate>
37+
38+
<!-- Need to Specify transaction weights for each workload .. Otherwise the number of fields won't match -->
39+
<weights bench="tpcc">45,43,4,4,4</weights>
40+
<weights bench="chbenchmark">3, 2, 3, 2, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5</weights>
41+
</work>
42+
43+
<!-- Extra features showcase -->
44+
<!-- <work> -->
45+
<!-- <time>60</time> -->
46+
47+
<!-- <rate>200</rate> -->
48+
<!-- <rate bench="chbenchmark">disabled</rate> -->
49+
50+
<!-- NOTE: TPCC workers won't be distributed evenly between warehouses if not all workers are active -->
51+
<!-- <active_terminals>1</active_terminals> -->
52+
<!-- <active_terminals bench="chbenchmark">1</active_terminals> -->
53+
54+
<!-- Specifies transaction weight for each workload. -->
55+
<!-- <weights bench="tpcc">45,43,4,4,4</weights> -->
56+
<!-- <weights bench="chbenchmark">3, 2, 3, 2, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5</weights> -->
57+
<!-- </work> -->
58+
59+
<!--
60+
<work>
61+
<time>60</time>
62+
63+
<rate>100</rate>
64+
<rate bench="chbenchmark">unlimited</rate>
65+
66+
<weights bench="tpcc">45,43,4,4,4</weights>
67+
<weights bench="chbenchmark">3, 2, 3, 2, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5</weights>
68+
</work>
69+
-->
70+
</works>
71+
72+
73+
<!-- CH specific -->
74+
<transactiontypes bench="chbenchmark">
75+
<transactiontype>
76+
<name>Q1</name>
77+
</transactiontype>
78+
<transactiontype>
79+
<name>Q2</name>
80+
</transactiontype>
81+
<transactiontype>
82+
<name>Q3</name>
83+
</transactiontype>
84+
<transactiontype>
85+
<name>Q4</name>
86+
</transactiontype>
87+
<transactiontype>
88+
<name>Q5</name>
89+
</transactiontype>
90+
<transactiontype>
91+
<name>Q6</name>
92+
</transactiontype>
93+
<transactiontype>
94+
<name>Q7</name>
95+
</transactiontype>
96+
<transactiontype>
97+
<name>Q8</name>
98+
</transactiontype>
99+
<transactiontype>
100+
<name>Q9</name>
101+
</transactiontype>
102+
<transactiontype>
103+
<name>Q10</name>
104+
</transactiontype>
105+
<transactiontype>
106+
<name>Q11</name>
107+
</transactiontype>
108+
<transactiontype>
109+
<name>Q12</name>
110+
</transactiontype>
111+
<transactiontype>
112+
<name>Q13</name>
113+
</transactiontype>
114+
<transactiontype>
115+
<name>Q14</name>
116+
</transactiontype>
117+
<transactiontype>
118+
<name>Q15</name>
119+
</transactiontype>
120+
<transactiontype>
121+
<name>Q16</name>
122+
</transactiontype>
123+
<transactiontype>
124+
<name>Q17</name>
125+
</transactiontype>
126+
<transactiontype>
127+
<name>Q18</name>
128+
</transactiontype>
129+
<transactiontype>
130+
<name>Q19</name>
131+
</transactiontype>
132+
<transactiontype>
133+
<name>Q20</name>
134+
</transactiontype>
135+
<transactiontype>
136+
<name>Q21</name>
137+
</transactiontype>
138+
<transactiontype>
139+
<name>Q22</name>
140+
</transactiontype>
141+
</transactiontypes>
142+
143+
<!-- TPCC specific -->
144+
<transactiontypes bench="tpcc">
145+
<transactiontype>
146+
<name>NewOrder</name>
147+
</transactiontype>
148+
<transactiontype>
149+
<name>Payment</name>
150+
</transactiontype>
151+
<transactiontype>
152+
<name>OrderStatus</name>
153+
</transactiontype>
154+
<transactiontype>
155+
<name>Delivery</name>
156+
</transactiontype>
157+
<transactiontype>
158+
<name>StockLevel</name>
159+
</transactiontype>
160+
</transactiontypes>
161+
</parameters>

‎config/sqlite/sample_epinions_config.xml

+2-2
Original file line numberDiff line numberDiff line change
@@ -4,13 +4,13 @@
44
<!-- Connection details -->
55
<type>SQLITE</type>
66
<driver>org.sqlite.JDBC</driver>
7-
<url>jdbc:sqlite:resourcestresser.db</url>
7+
<url>jdbc:sqlite:epinions.db</url>
88
<isolation>TRANSACTION_SERIALIZABLE</isolation>
99
<batchsize>128</batchsize>
1010

1111
<!-- Scalefactor in Epinions scales by *2000 the number of users-->
1212
<scalefactor>0.1</scalefactor>
13-
13+
1414
<!-- SQLITE only supports one writer thread -->
1515
<loaderThreads>1</loaderThreads>
1616

‎docker/build-run-benchmark-with-docker.sh

+16-7
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,7 @@ if [ "$BENCHBASE_PROFILE" == 'sqlite' ]; then
3333
fi
3434
EXTRA_DOCKER_ARGS="-v $SRC_DIR/$benchmark.db:/benchbase/profiles/sqlite/$benchmark.db"
3535

36-
if [ "$benchmark" == 'templated' ]; then
36+
if echo "$benchmark" | egrep -qx '(templated|chbenchmark)'; then
3737
# See notes below:
3838
EXTRA_DOCKER_ARGS+=" -v $SRC_DIR/$benchmark.db:/benchbase/profiles/sqlite/tpcc.db"
3939
fi
@@ -49,7 +49,7 @@ if [ "${SKIP_LOAD_DB:-false}" != 'true' ]; then
4949
# For templated benchmarks, we need to preload some data for the test since by
5050
# design, templated benchmarks do not support the 'load' operation
5151
# In this case, we load the tpcc data.
52-
if [ "$benchmark" == 'templated' ]; then
52+
if echo "$benchmark" | egrep -qx '(templated|chbenchmark)'; then
5353
load_benchmark='tpcc'
5454

5555
echo "INFO: Loading tpcc data for templated benchmark"
@@ -59,15 +59,24 @@ if [ "${SKIP_LOAD_DB:-false}" != 'true' ]; then
5959
else
6060
config="config/sample_tpcc_config.xml"
6161
fi
62-
else
62+
63+
BUILD_IMAGE=false EXTRA_DOCKER_ARGS="--network=host $EXTRA_DOCKER_ARGS" \
64+
./docker/benchbase/run-full-image.sh \
65+
--config "$config" --bench "$load_benchmark" \
66+
--create=true --load=true --execute=false
67+
fi
68+
69+
# For chbenchmark, we also load it's data in addition to tpcc.
70+
if ! echo "$benchmark" | egrep -qx '(templated)'; then
6371
echo "INFO: Loading $benchmark data"
6472
load_benchmark="$benchmark"
6573
config="config/sample_${benchmark}_config.xml"
74+
75+
BUILD_IMAGE=false EXTRA_DOCKER_ARGS="--network=host $EXTRA_DOCKER_ARGS" \
76+
./docker/benchbase/run-full-image.sh \
77+
--config "$config" --bench "$load_benchmark" \
78+
--create=true --load=true --execute=false
6679
fi
67-
BUILD_IMAGE=false EXTRA_DOCKER_ARGS="--network=host $EXTRA_DOCKER_ARGS" \
68-
./docker/benchbase/run-full-image.sh \
69-
--config "$config" --bench "$load_benchmark" \
70-
--create=true --load=true --execute=false
7180
else
7281
echo "INFO: Skipping load of $benchmark data"
7382
fi

‎src/main/resources/benchmarks/chbenchmark/ddl-sqlite.sql

+31
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
DROP TABLE IF EXISTS supplier;
2+
DROP TABLE IF EXISTS nation;
3+
DROP TABLE IF EXISTS region;
4+
5+
create table region
6+
(
7+
r_regionkey int not null,
8+
r_name char(55) not null,
9+
r_comment char(152) not null,
10+
PRIMARY KEY (r_regionkey)
11+
);
12+
13+
create table nation
14+
(
15+
n_nationkey int not null,
16+
n_name char(25) not null,
17+
n_regionkey int not null references region(r_regionkey) ON DELETE CASCADE,
18+
n_comment char(152) not null,
19+
PRIMARY KEY ( n_nationkey )
20+
);
21+
22+
create table supplier (
23+
su_suppkey int not null,
24+
su_name char(25) not null,
25+
su_address varchar(40) not null,
26+
su_nationkey int not null references nation(n_nationkey) ON DELETE CASCADE,
27+
su_phone char(15) not null,
28+
su_acctbal numeric(12,2) not null,
29+
su_comment char(101) not null,
30+
PRIMARY KEY ( su_suppkey )
31+
);

0 commit comments

Comments
 (0)