Skip to content

Commit e6bb999

Browse files
filipecosta90filipecosta90
authored
Fixed wrong redis dependencies building due to hdr_histogram not existing on < 6.2.X (#58)
* [fix] Fixed wrong redis dependencies building due to hdr_histogram not existing on < 6.2.X * [fix] Fixed wrong identation on cli test * [wip] including watchdog to generate build/run events overall time-series * [fix] Fixed unused deps on watchdog
1 parent 6270f65 commit e6bb999

File tree

13 files changed

+458
-80
lines changed

13 files changed

+458
-80
lines changed

‎poetry.lock

Lines changed: 1 addition & 1 deletion
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎pyproject.toml

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
[tool.poetry]
22
name = "redis-benchmarks-specification"
3-
version = "0.1.16"
3+
version = "0.1.17"
44
description = "The Redis benchmarks specification describes the cross-language/tools requirements and expectations to foster performance and observability standards around redis related technologies. Members from both industry and academia, including organizations and individuals are encouraged to contribute."
55
authors = ["filipecosta90 <filipecosta.90@gmail.com>","Redis Performance Group <performance@redis.com>"]
66
readme = "Readme.md"
@@ -43,3 +43,4 @@ redis-benchmarks-spec-api = "redis_benchmarks_specification.__api__.api:main"
4343
redis-benchmarks-spec-builder = "redis_benchmarks_specification.__builder__.builder:main"
4444
redis-benchmarks-spec-sc-coordinator = "redis_benchmarks_specification.__self_contained_coordinator__.self_contained_coordinator:main"
4545
redis-benchmarks-spec-cli = "redis_benchmarks_specification.__cli__.cli:main"
46+
redis-benchmarks-spec-watchdog = "redis_benchmarks_specification.__watchdog__.watchdog:main"

‎redis_benchmarks_specification/__builder__/builder.py

Lines changed: 12 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@
2626
STREAM_KEYNAME_NEW_BUILD_EVENTS,
2727
REDIS_HEALTH_CHECK_INTERVAL,
2828
REDIS_SOCKET_TIMEOUT,
29+
REDIS_BINS_EXPIRE_SECS,
2930
)
3031
from redis_benchmarks_specification.__common__.package import (
3132
populate_with_poetry_data,
@@ -231,21 +232,22 @@ def builder_process_stream(builders_folder, conn, different_build_specs, previou
231232
z = ZipFileWithPermissions(io.BytesIO(buffer))
232233
z.extractall(temporary_dir)
233234
redis_dir = os.listdir(temporary_dir + "/")[0]
235+
deps_dir = os.listdir(temporary_dir + "/" + redis_dir + "/deps")
236+
deps_list = [
237+
"hiredis",
238+
"jemalloc",
239+
"linenoise",
240+
"lua",
241+
]
242+
if "hdr_histogram" in deps_dir:
243+
deps_list.append("hdr_histogram")
234244
redis_temporary_dir = temporary_dir + "/" + redis_dir + "/"
235245
logging.info("Using redis temporary dir {}".format(redis_temporary_dir))
236246
build_command = 'bash -c "make Makefile.dep {} && cd ./deps && CXX={} CC={} make {} {} -j && cd .. && CXX={} CC={} make {} {} -j"'.format(
237247
build_vars_str,
238248
cpp_compiler,
239249
compiler,
240-
" ".join(
241-
[
242-
"hdr_histogram",
243-
"hiredis",
244-
"jemalloc",
245-
"linenoise",
246-
"lua",
247-
]
248-
),
250+
" ".join(deps_list),
249251
build_vars_str,
250252
cpp_compiler,
251253
compiler,
@@ -291,8 +293,7 @@ def builder_process_stream(builders_folder, conn, different_build_specs, previou
291293
bin_artifact = open(
292294
"{}src/{}".format(redis_temporary_dir, artifact), "rb"
293295
).read()
294-
ttl = 24 * 7 * 60 * 60
295-
conn.set(bin_key, bytes(bin_artifact), ex=ttl)
296+
conn.set(bin_key, bytes(bin_artifact), ex=REDIS_BINS_EXPIRE_SECS)
296297
build_stream_fields[artifact] = bin_key
297298
build_stream_fields["{}_len_bytes".format(artifact)] = len(
298299
bytes(bin_artifact)

‎redis_benchmarks_specification/__cli__/args.py

Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,66 @@
1+
# BSD 3-Clause License
2+
#
3+
# Copyright (c) 2021., Redis Labs Modules
4+
# All rights reserved.
5+
#
6+
import datetime
7+
8+
9+
from redis_benchmarks_specification.__common__.env import (
10+
GH_REDIS_SERVER_HOST,
11+
GH_TOKEN,
12+
GH_REDIS_SERVER_PORT,
13+
GH_REDIS_SERVER_AUTH,
14+
GH_REDIS_SERVER_USER,
15+
)
16+
17+
from redisbench_admin.run.common import get_start_time_vars
18+
19+
START_TIME_NOW_UTC, _, _ = get_start_time_vars()
20+
START_TIME_LAST_YEAR_UTC = START_TIME_NOW_UTC - datetime.timedelta(days=7)
21+
22+
23+
def spec_cli_args(parser):
24+
parser.add_argument("--redis_host", type=str, default=GH_REDIS_SERVER_HOST)
25+
parser.add_argument("--branch", type=str, default="unstable")
26+
parser.add_argument("--gh_token", type=str, default=GH_TOKEN)
27+
parser.add_argument("--redis_port", type=int, default=GH_REDIS_SERVER_PORT)
28+
parser.add_argument("--redis_pass", type=str, default=GH_REDIS_SERVER_AUTH)
29+
parser.add_argument("--redis_user", type=str, default=GH_REDIS_SERVER_USER)
30+
parser.add_argument(
31+
"--from-date",
32+
type=lambda s: datetime.datetime.strptime(s, "%Y-%m-%d"),
33+
default=START_TIME_LAST_YEAR_UTC,
34+
)
35+
parser.add_argument(
36+
"--to-date",
37+
type=lambda s: datetime.datetime.strptime(s, "%Y-%m-%d"),
38+
default=START_TIME_NOW_UTC,
39+
)
40+
parser.add_argument("--redis_repo", type=str, default=None)
41+
parser.add_argument("--trigger-unstable-commits", type=bool, default=True)
42+
parser.add_argument(
43+
"--use-tags",
44+
default=False,
45+
action="store_true",
46+
help="Iterate over the git tags.",
47+
)
48+
parser.add_argument(
49+
"--tags-regexp",
50+
type=str,
51+
default=".*",
52+
help="Interpret PATTERN as a regular expression to filter tag names",
53+
)
54+
parser.add_argument(
55+
"--use-branch",
56+
default=False,
57+
action="store_true",
58+
help="Iterate over the git commits.",
59+
)
60+
parser.add_argument(
61+
"--dry-run",
62+
default=False,
63+
action="store_true",
64+
help="Only check how many benchmarks we would trigger. Don't request benchmark runs at the end.",
65+
)
66+
return parser

‎redis_benchmarks_specification/__cli__/cli.py

Lines changed: 57 additions & 66 deletions
Original file line numberDiff line numberDiff line change
@@ -7,40 +7,35 @@
77
import argparse
88
import datetime
99
import logging
10+
import re
1011
import shutil
1112
import subprocess
13+
import sys
1214
import tempfile
1315
import git
1416
import packaging
1517
import redis
1618
from packaging import version
1719

18-
# logging settings
19-
from redisbench_admin.cli import populate_with_poetry_data
20-
from redisbench_admin.run.common import get_start_time_vars
2120

21+
from redis_benchmarks_specification.__cli__.args import spec_cli_args
2222
from redis_benchmarks_specification.__common__.builder_schema import (
2323
get_commit_dict_from_sha,
2424
request_build_from_commit_info,
2525
)
26-
from redis_benchmarks_specification.__common__.env import (
27-
GH_REDIS_SERVER_HOST,
28-
GH_REDIS_SERVER_AUTH,
29-
GH_REDIS_SERVER_USER,
30-
GH_REDIS_SERVER_PORT,
31-
GH_TOKEN,
26+
from redis_benchmarks_specification.__common__.env import REDIS_BINS_EXPIRE_SECS
27+
from redis_benchmarks_specification.__common__.package import (
28+
get_version_string,
29+
populate_with_poetry_data,
3230
)
33-
from redis_benchmarks_specification.__common__.package import get_version_string
3431

32+
# logging settings
3533
logging.basicConfig(
3634
format="%(asctime)s %(levelname)-4s %(message)s",
3735
level=logging.INFO,
3836
datefmt="%Y-%m-%d %H:%M:%S",
3937
)
4038

41-
START_TIME_NOW_UTC, _, _ = get_start_time_vars()
42-
START_TIME_LAST_YEAR_UTC = START_TIME_NOW_UTC - datetime.timedelta(days=7)
43-
4439

4540
def main():
4641
_, _, project_version = populate_with_poetry_data()
@@ -49,43 +44,21 @@ def main():
4944
description=get_version_string(project_name, project_version),
5045
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
5146
)
52-
parser.add_argument("--redis_host", type=str, default=GH_REDIS_SERVER_HOST)
53-
parser.add_argument("--branch", type=str, default="unstable")
54-
parser.add_argument("--gh_token", type=str, default=GH_TOKEN)
55-
parser.add_argument("--redis_port", type=int, default=GH_REDIS_SERVER_PORT)
56-
parser.add_argument("--redis_pass", type=str, default=GH_REDIS_SERVER_AUTH)
57-
parser.add_argument("--redis_user", type=str, default=GH_REDIS_SERVER_USER)
58-
parser.add_argument(
59-
"--from-date",
60-
type=lambda s: datetime.datetime.strptime(s, "%Y-%m-%d"),
61-
default=START_TIME_LAST_YEAR_UTC,
62-
)
63-
parser.add_argument(
64-
"--to-date",
65-
type=lambda s: datetime.datetime.strptime(s, "%Y-%m-%d"),
66-
default=START_TIME_NOW_UTC,
67-
)
68-
parser.add_argument("--redis_repo", type=str, default=None)
69-
parser.add_argument("--trigger-unstable-commits", type=bool, default=True)
70-
parser.add_argument(
71-
"--use-tags",
72-
default=False,
73-
action="store_true",
74-
help="Iterate over the git tags.",
75-
)
76-
parser.add_argument(
77-
"--use-commits",
78-
default=False,
79-
action="store_true",
80-
help="Iterate over the git commits.",
81-
)
82-
parser.add_argument(
83-
"--dry-run",
84-
default=False,
85-
action="store_true",
86-
help="Only check how many benchmarks we would trigger. Don't request benchmark runs at the end.",
87-
)
47+
parser = spec_cli_args(parser)
8848
args = parser.parse_args()
49+
50+
cli_command_logic(args, project_name, project_version)
51+
52+
53+
def cli_command_logic(args, project_name, project_version):
54+
logging.info(
55+
"Using: {project_name} {project_version}".format(
56+
project_name=project_name, project_version=project_version
57+
)
58+
)
59+
if args.use_branch is False and args.use_tags is False:
60+
logging.error("You must specify either --use-tags or --use-branch flag")
61+
sys.exit(1)
8962
redisDirPath = args.redis_repo
9063
cleanUp = False
9164
if redisDirPath is None:
@@ -115,9 +88,8 @@ def main():
11588
)
11689
)
11790
repo = git.Repo(redisDirPath)
118-
11991
commits = []
120-
if args.use_commits:
92+
if args.use_branch:
12193
for commit in repo.iter_commits():
12294
if (
12395
args.from_date
@@ -129,6 +101,17 @@ def main():
129101
print(commit.summary)
130102
commits.append({"git_hash": commit.hexsha, "git_branch": args.branch})
131103
if args.use_tags:
104+
tags_regexp = args.tags_regexp
105+
if tags_regexp == ".*":
106+
logging.info(
107+
"Acception all tags that follow semver between the timeframe. If you need further filter specify a regular expression via --tags-regexp"
108+
)
109+
else:
110+
logging.info(
111+
"Filtering all tags via a regular expression: {}".format(tags_regexp)
112+
)
113+
tags_regex_string = re.compile(tags_regexp)
114+
132115
tags = sorted(repo.tags, key=lambda t: t.commit.committed_datetime)
133116
for tag in tags:
134117
if (
@@ -141,33 +124,38 @@ def main():
141124

142125
try:
143126
version.Version(tag.name)
144-
git_version = tag.name
145-
print(
146-
"Commit summary: {}. Extract semver: {}".format(
147-
tag.commit.summary, git_version
127+
match_obj = re.search(tags_regex_string, tag.name)
128+
if match_obj is None:
129+
logging.info(
130+
"Skipping {} given it does not match regex {}".format(
131+
tag.name, tags_regexp
132+
)
133+
)
134+
else:
135+
git_version = tag.name
136+
print(
137+
"Commit summary: {}. Extract semver: {}".format(
138+
tag.commit.summary, git_version
139+
)
140+
)
141+
commits.append(
142+
{"git_hash": tag.commit.hexsha, "git_version": git_version}
148143
)
149-
)
150-
# head = repo.lookup_reference(tag.commit).resolve()
151-
commits.append(
152-
{"git_hash": tag.commit.hexsha, "git_version": git_version}
153-
)
154144
except packaging.version.InvalidVersion:
155145
logging.info(
156146
"Ignoring tag {} given we were not able to extract commit or version info from it.".format(
157147
tag.name
158148
)
159149
)
160150
pass
161-
162151
by_description = "n/a"
163-
if args.use_commits:
152+
if args.use_branch:
164153
by_description = "from branch {}".format(args.branch)
165154
if args.use_tags:
166155
by_description = "by tags"
167156
logging.info(
168157
"Will trigger {} distinct tests {}.".format(len(commits), by_description)
169158
)
170-
171159
if args.dry_run is False:
172160
conn = redis.StrictRedis(
173161
host=args.redis_host,
@@ -188,10 +176,14 @@ def main():
188176
) = get_commit_dict_from_sha(
189177
cdict["git_hash"], "redis", "redis", cdict, True, args.gh_token
190178
)
191-
binary_exp_secs = 24 * 7 * 60 * 60
192179
if result is True:
193180
result, reply_fields, error_msg = request_build_from_commit_info(
194-
conn, commit_dict, {}, binary_key, binary_value, binary_exp_secs
181+
conn,
182+
commit_dict,
183+
{},
184+
binary_key,
185+
binary_value,
186+
REDIS_BINS_EXPIRE_SECS,
195187
)
196188
logging.info(
197189
"Successfully requested a build for commit: {}. Request stream id: {}.".format(
@@ -203,7 +195,6 @@ def main():
203195

204196
else:
205197
logging.info("Skipping actual work trigger ( dry-run )")
206-
207198
if cleanUp is True:
208199
logging.info("Removing temporary redis dir {}.".format(redisDirPath))
209200
shutil.rmtree(redisDirPath)

‎redis_benchmarks_specification/__common__/env.py

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,9 @@
4848
REDIS_AUTH_SERVER_PORT = int(os.getenv("REDIS_AUTH_SERVER_PORT", "6380"))
4949
REDIS_HEALTH_CHECK_INTERVAL = int(os.getenv("REDIS_HEALTH_CHECK_INTERVAL", "15"))
5050
REDIS_SOCKET_TIMEOUT = int(os.getenv("REDIS_SOCKET_TIMEOUT", "300"))
51+
REDIS_BINS_EXPIRE_SECS = int(
52+
os.getenv("REDIS_BINS_EXPIRE_SECS", "{}".format(24 * 7 * 60 * 60))
53+
)
5154

5255
# environment variables
5356
PULL_REQUEST_TRIGGER_LABEL = os.getenv(

‎redis_benchmarks_specification/__watchdog__/__init__.py

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
# Apache 2 License
2+
#
3+
# Copyright (c) 2021., Redis Labs
4+
# All rights reserved.
5+
#

0 commit comments

Comments
 (0)