Package coprs :: Module helpers
[hide private]
[frames] | no frames]

Source Code for Module coprs.helpers

  1  from __future__ import division 
  2   
  3  import math 
  4  import random 
  5  import string 
  6   
  7  from six import with_metaclass 
  8  from six.moves.urllib.parse import urljoin, urlparse 
  9  import pipes 
 10  from textwrap import dedent 
 11  import re 
 12   
 13  import flask 
 14  from flask import url_for 
 15  from dateutil import parser as dt_parser 
 16  from netaddr import IPAddress, IPNetwork 
 17  from redis import StrictRedis 
 18  from sqlalchemy.types import TypeDecorator, VARCHAR 
 19  import json 
 20   
 21  from coprs import constants 
 22  from coprs import app 
23 24 25 -def generate_api_token(size=30):
26 """ Generate a random string used as token to access the API 27 remotely. 28 29 :kwarg: size, the size of the token to generate, defaults to 30 30 chars. 31 :return: a string, the API token for the user. 32 """ 33 return ''.join(random.choice(string.ascii_lowercase) for x in range(size))
34 35 36 REPO_DL_STAT_FMT = "repo_dl_stat::{copr_user}@{copr_project_name}:{copr_name_release}" 37 CHROOT_REPO_MD_DL_STAT_FMT = "chroot_repo_metadata_dl_stat:hset::{copr_user}@{copr_project_name}:{copr_chroot}" 38 CHROOT_RPMS_DL_STAT_FMT = "chroot_rpms_dl_stat:hset::{copr_user}@{copr_project_name}:{copr_chroot}" 39 PROJECT_RPMS_DL_STAT_FMT = "project_rpms_dl_stat:hset::{copr_user}@{copr_project_name}"
40 41 42 -class CounterStatType(object):
43 REPO_DL = "repo_dl"
44
45 46 -class EnumType(type):
47
48 - def __call__(self, attr):
49 if isinstance(attr, int): 50 for k, v in self.vals.items(): 51 if v == attr: 52 return k 53 raise KeyError("num {0} is not mapped".format(attr)) 54 else: 55 return self.vals[attr]
56
57 58 -class PermissionEnum(with_metaclass(EnumType, object)):
59 vals = {"nothing": 0, "request": 1, "approved": 2} 60 61 @classmethod
62 - def choices_list(cls, without=-1):
63 return [(n, k) for k, n in cls.vals.items() if n != without]
64
65 66 -class ActionTypeEnum(with_metaclass(EnumType, object)):
67 vals = { 68 "delete": 0, 69 "rename": 1, 70 "legal-flag": 2, 71 "createrepo": 3, 72 "update_comps": 4, 73 "gen_gpg_key": 5, 74 "rawhide_to_release": 6, 75 "fork": 7, 76 "update_module_md": 8, 77 "build_module": 9, 78 "cancel_build": 10, 79 }
80
81 82 -class BackendResultEnum(with_metaclass(EnumType, object)):
83 vals = {"waiting": 0, "success": 1, "failure": 2}
84
85 86 -class RoleEnum(with_metaclass(EnumType, object)):
87 vals = {"user": 0, "admin": 1}
88
89 90 -class StatusEnum(with_metaclass(EnumType, object)):
91 vals = {"failed": 0, 92 "succeeded": 1, 93 "canceled": 2, 94 "running": 3, 95 "pending": 4, 96 "skipped": 5, # if there was this package built already 97 "starting": 6, # build picked by worker but no VM initialized 98 "importing": 7, # SRPM is being imported to dist-git 99 "forked": 8, # build(-chroot) was forked 100 "unknown": 1000, # order_to_status/status_to_order issue 101 }
102
103 104 -class ModuleStatusEnum(with_metaclass(EnumType, object)):
105 vals = {"pending": 0, "succeeded": 1, "failed": 2}
106
107 108 -class BuildSourceEnum(with_metaclass(EnumType, object)):
109 vals = {"unset": 0, 110 "srpm_link": 1, # url 111 "srpm_upload": 2, # pkg, tmp 112 "git_and_tito": 3, # git_url, git_dir, git_branch, tito_test 113 "mock_scm": 4, # scm_type, scm_url, spec, scm_branch 114 "pypi": 5, # package_name, version, python_versions 115 "rubygems": 6, # gem_name 116 "distgit": 7, # url, branch 117 }
118
119 120 # The same enum is also in distgit's helpers.py 121 -class FailTypeEnum(with_metaclass(EnumType, object)):
122 vals = {"unset": 0, 123 # General errors mixed with errors for SRPM URL/upload: 124 "unknown_error": 1, 125 "build_error": 2, 126 "srpm_import_failed": 3, 127 "srpm_download_failed": 4, 128 "srpm_query_failed": 5, 129 "import_timeout_exceeded": 6, 130 # Git and Tito errors: 131 "tito_general_error": 30, 132 "git_clone_failed": 31, 133 "git_wrong_directory": 32, 134 "git_checkout_error": 33, 135 "srpm_build_error": 34, 136 }
137
138 139 -class JSONEncodedDict(TypeDecorator):
140 """Represents an immutable structure as a json-encoded string. 141 142 Usage:: 143 144 JSONEncodedDict(255) 145 146 """ 147 148 impl = VARCHAR 149
150 - def process_bind_param(self, value, dialect):
151 if value is not None: 152 value = json.dumps(value) 153 154 return value
155
156 - def process_result_value(self, value, dialect):
157 if value is not None: 158 value = json.loads(value) 159 return value
160
161 -class Paginator(object):
162
163 - def __init__(self, query, total_count, page=1, 164 per_page_override=None, urls_count_override=None, 165 additional_params=None):
166 167 self.query = query 168 self.total_count = total_count 169 self.page = page 170 self.per_page = per_page_override or constants.ITEMS_PER_PAGE 171 self.urls_count = urls_count_override or constants.PAGES_URLS_COUNT 172 self.additional_params = additional_params or dict() 173 174 self._sliced_query = None
175
176 - def page_slice(self, page):
177 return (self.per_page * (page - 1), 178 self.per_page * page)
179 180 @property
181 - def sliced_query(self):
182 if not self._sliced_query: 183 self._sliced_query = self.query[slice(*self.page_slice(self.page))] 184 return self._sliced_query
185 186 @property
187 - def pages(self):
188 return int(math.ceil(self.total_count / float(self.per_page)))
189
190 - def border_url(self, request, start):
191 if start: 192 if self.page - 1 > self.urls_count // 2: 193 return self.url_for_other_page(request, 1), 1 194 else: 195 if self.page < self.pages - self.urls_count // 2: 196 return self.url_for_other_page(request, self.pages), self.pages 197 198 return None
199
200 - def get_urls(self, request):
201 left_border = self.page - self.urls_count // 2 202 left_border = 1 if left_border < 1 else left_border 203 right_border = self.page + self.urls_count // 2 204 right_border = self.pages if right_border > self.pages else right_border 205 206 return [(self.url_for_other_page(request, i), i) 207 for i in range(left_border, right_border + 1)]
208
209 - def url_for_other_page(self, request, page):
210 args = request.view_args.copy() 211 args["page"] = page 212 args.update(self.additional_params) 213 return flask.url_for(request.endpoint, **args)
214
215 216 -def chroot_to_branch(chroot):
217 """ 218 Get a git branch name from chroot. Follow the fedora naming standard. 219 """ 220 os, version, arch = chroot.split("-") 221 if os == "fedora": 222 if version == "rawhide": 223 return "master" 224 os = "f" 225 elif os == "epel" and int(version) <= 6: 226 os = "el" 227 elif os == "mageia" and version == "cauldron": 228 os = "cauldron" 229 version = "" 230 elif os == "mageia": 231 os = "mga" 232 return "{}{}".format(os, version)
233
234 235 -def splitFilename(filename):
236 """ 237 Pass in a standard style rpm fullname 238 239 Return a name, version, release, epoch, arch, e.g.:: 240 foo-1.0-1.i386.rpm returns foo, 1.0, 1, i386 241 1:bar-9-123a.ia64.rpm returns bar, 9, 123a, 1, ia64 242 """ 243 244 if filename[-4:] == '.rpm': 245 filename = filename[:-4] 246 247 archIndex = filename.rfind('.') 248 arch = filename[archIndex+1:] 249 250 relIndex = filename[:archIndex].rfind('-') 251 rel = filename[relIndex+1:archIndex] 252 253 verIndex = filename[:relIndex].rfind('-') 254 ver = filename[verIndex+1:relIndex] 255 256 epochIndex = filename.find(':') 257 if epochIndex == -1: 258 epoch = '' 259 else: 260 epoch = filename[:epochIndex] 261 262 name = filename[epochIndex + 1:verIndex] 263 return name, ver, rel, epoch, arch
264
265 266 -def parse_package_name(pkg):
267 """ 268 Parse package name from possibly incomplete nvra string. 269 """ 270 271 if pkg.count(".") >= 3 and pkg.count("-") >= 2: 272 return splitFilename(pkg)[0] 273 274 # doesn"t seem like valid pkg string, try to guess package name 275 result = "" 276 pkg = pkg.replace(".rpm", "").replace(".src", "") 277 278 for delim in ["-", "."]: 279 if delim in pkg: 280 parts = pkg.split(delim) 281 for part in parts: 282 if any(map(lambda x: x.isdigit(), part)): 283 return result[:-1] 284 285 result += part + "-" 286 287 return result[:-1] 288 289 return pkg
290
291 292 -def generate_repo_url(mock_chroot, url):
293 """ Generates url with build results for .repo file. 294 No checks if copr or mock_chroot exists. 295 """ 296 if mock_chroot.os_release == "fedora": 297 if mock_chroot.os_version != "rawhide": 298 mock_chroot.os_version = "$releasever" 299 300 url = urljoin( 301 url, "{0}-{1}-{2}/".format(mock_chroot.os_release, 302 mock_chroot.os_version, "$basearch")) 303 304 return url
305
306 307 -def fix_protocol_for_backend(url):
308 """ 309 Ensure that url either has http or https protocol according to the 310 option in app config "ENFORCE_PROTOCOL_FOR_BACKEND_URL" 311 """ 312 if app.config["ENFORCE_PROTOCOL_FOR_BACKEND_URL"] == "https": 313 return url.replace("http://", "https://") 314 elif app.config["ENFORCE_PROTOCOL_FOR_BACKEND_URL"] == "http": 315 return url.replace("https://", "http://") 316 else: 317 return url
318
319 320 -def fix_protocol_for_frontend(url):
321 """ 322 Ensure that url either has http or https protocol according to the 323 option in app config "ENFORCE_PROTOCOL_FOR_FRONTEND_URL" 324 """ 325 if app.config["ENFORCE_PROTOCOL_FOR_FRONTEND_URL"] == "https": 326 return url.replace("http://", "https://") 327 elif app.config["ENFORCE_PROTOCOL_FOR_FRONTEND_URL"] == "http": 328 return url.replace("https://", "http://") 329 else: 330 return url
331
332 333 -class Serializer(object):
334
335 - def to_dict(self, options=None):
336 """ 337 Usage: 338 339 SQLAlchObject.to_dict() => returns a flat dict of the object 340 SQLAlchObject.to_dict({"foo": {}}) => returns a dict of the object 341 and will include a flat dict of object foo inside of that 342 SQLAlchObject.to_dict({"foo": {"bar": {}}, "spam": {}}) => returns 343 a dict of the object, which will include dict of foo 344 (which will include dict of bar) and dict of spam. 345 346 Options can also contain two special values: __columns_only__ 347 and __columns_except__ 348 349 If present, the first makes only specified fiels appear, 350 the second removes specified fields. Both of these fields 351 must be either strings (only works for one field) or lists 352 (for one and more fields). 353 354 SQLAlchObject.to_dict({"foo": {"__columns_except__": ["id"]}, 355 "__columns_only__": "name"}) => 356 357 The SQLAlchObject will only put its "name" into the resulting dict, 358 while "foo" all of its fields except "id". 359 360 Options can also specify whether to include foo_id when displaying 361 related foo object (__included_ids__, defaults to True). 362 This doesn"t apply when __columns_only__ is specified. 363 """ 364 365 result = {} 366 if options is None: 367 options = {} 368 columns = self.serializable_attributes 369 370 if "__columns_only__" in options: 371 columns = options["__columns_only__"] 372 else: 373 columns = set(columns) 374 if "__columns_except__" in options: 375 columns_except = options["__columns_except__"] 376 if not isinstance(options["__columns_except__"], list): 377 columns_except = [options["__columns_except__"]] 378 379 columns -= set(columns_except) 380 381 if ("__included_ids__" in options and 382 options["__included_ids__"] is False): 383 384 related_objs_ids = [ 385 r + "_id" for r, _ in options.items() 386 if not r.startswith("__")] 387 388 columns -= set(related_objs_ids) 389 390 columns = list(columns) 391 392 for column in columns: 393 result[column] = getattr(self, column) 394 395 for related, values in options.items(): 396 if hasattr(self, related): 397 result[related] = getattr(self, related).to_dict(values) 398 return result
399 400 @property
401 - def serializable_attributes(self):
402 return map(lambda x: x.name, self.__table__.columns)
403
404 405 -class RedisConnectionProvider(object):
406 - def __init__(self, config):
407 self.host = config.get("REDIS_HOST", "127.0.0.1") 408 self.port = int(config.get("REDIS_PORT", "6379"))
409
410 - def get_connection(self):
411 return StrictRedis(host=self.host, port=self.port)
412
413 414 -def get_redis_connection():
415 """ 416 Creates connection to redis, now we use default instance at localhost, no config needed 417 """ 418 return StrictRedis()
419
420 421 -def dt_to_unixtime(dt):
422 """ 423 Converts datetime to unixtime 424 :param dt: DateTime instance 425 :rtype: float 426 """ 427 return float(dt.strftime('%s'))
428
429 430 -def string_dt_to_unixtime(dt_string):
431 """ 432 Converts datetime to unixtime from string 433 :param dt_string: datetime string 434 :rtype: str 435 """ 436 return dt_to_unixtime(dt_parser.parse(dt_string))
437
438 439 -def is_ip_from_builder_net(ip):
440 """ 441 Checks is ip is owned by the builders network 442 :param str ip: IPv4 address 443 :return bool: True 444 """ 445 ip_addr = IPAddress(ip) 446 for subnet in app.config.get("BUILDER_IPS", ["127.0.0.1/24"]): 447 if ip_addr in IPNetwork(subnet): 448 return True 449 450 return False
451
452 453 -def str2bool(v):
454 if v is None: 455 return False 456 return v.lower() in ("yes", "true", "t", "1")
457
458 459 -def copr_url(view, copr, **kwargs):
460 """ 461 Examine given copr and generate proper URL for the `view` 462 463 Values of `username/group_name` and `coprname` are automatically passed as the first two URL parameters, 464 and therefore you should *not* pass them manually. 465 466 Usage: 467 copr_url("coprs_ns.foo", copr) 468 copr_url("coprs_ns.foo", copr, arg1='bar', arg2='baz) 469 """ 470 if copr.is_a_group_project: 471 return url_for(view, group_name=copr.group.name, coprname=copr.name, **kwargs) 472 return url_for(view, username=copr.user.name, coprname=copr.name, **kwargs)
473
474 475 -def url_for_copr_view(view, group_view, copr, **kwargs):
476 if copr.is_a_group_project: 477 return url_for(group_view, group_name=copr.group.name, coprname=copr.name, **kwargs) 478 else: 479 return url_for(view, username=copr.user.name, coprname=copr.name, **kwargs)
480 481 482 from sqlalchemy.engine.default import DefaultDialect 483 from sqlalchemy.sql.sqltypes import String, DateTime, NullType 484 485 # python2/3 compatible. 486 PY3 = str is not bytes 487 text = str if PY3 else unicode 488 int_type = int if PY3 else (int, long) 489 str_type = str if PY3 else (str, unicode)
490 491 492 -class StringLiteral(String):
493 """Teach SA how to literalize various things."""
494 - def literal_processor(self, dialect):
495 super_processor = super(StringLiteral, self).literal_processor(dialect) 496 497 def process(value): 498 if isinstance(value, int_type): 499 return text(value) 500 if not isinstance(value, str_type): 501 value = text(value) 502 result = super_processor(value) 503 if isinstance(result, bytes): 504 result = result.decode(dialect.encoding) 505 return result
506 return process
507
508 509 -class LiteralDialect(DefaultDialect):
510 colspecs = { 511 # prevent various encoding explosions 512 String: StringLiteral, 513 # teach SA about how to literalize a datetime 514 DateTime: StringLiteral, 515 # don't format py2 long integers to NULL 516 NullType: StringLiteral, 517 }
518
519 520 -def literal_query(statement):
521 """NOTE: This is entirely insecure. DO NOT execute the resulting strings.""" 522 import sqlalchemy.orm 523 if isinstance(statement, sqlalchemy.orm.Query): 524 statement = statement.statement 525 return statement.compile( 526 dialect=LiteralDialect(), 527 compile_kwargs={'literal_binds': True}, 528 ).string
529
530 531 -def stream_template(template_name, **context):
532 app.update_template_context(context) 533 t = app.jinja_env.get_template(template_name) 534 rv = t.stream(context) 535 rv.enable_buffering(2) 536 return rv
537
538 539 -def generate_repo_name(repo_url):
540 """ based on url, generate repo name """ 541 repo_url = re.sub("[^a-zA-Z0-9]", '_', repo_url) 542 repo_url = re.sub("(__*)", '_', repo_url) 543 repo_url = re.sub("(_*$)|^_*", '', repo_url) 544 return repo_url
545
546 547 -def pre_process_repo_url(chroot, repo_url):
548 """ 549 Expands variables and sanitize repo url to be used for mock config 550 """ 551 parsed_url = urlparse(repo_url) 552 if parsed_url.scheme == "copr": 553 user = parsed_url.netloc 554 prj = parsed_url.path.split("/")[1] 555 repo_url = "/".join([ 556 flask.current_app.config["BACKEND_BASE_URL"], 557 "results", user, prj, chroot 558 ]) + "/" 559 560 repo_url = repo_url.replace("$chroot", chroot) 561 repo_url = repo_url.replace("$distname", chroot.split("-")[0]) 562 563 return pipes.quote(repo_url)
564
565 566 -def generate_build_config(copr, chroot_id):
567 """ Return dict with proper build config contents """ 568 chroot = None 569 for i in copr.copr_chroots: 570 if i.mock_chroot.name == chroot_id: 571 chroot = i 572 if not chroot: 573 return {} 574 575 packages = "" if not chroot.buildroot_pkgs else chroot.buildroot_pkgs 576 577 repos = [{ 578 "id": "copr_base", 579 "url": copr.repo_url + "/{}/".format(chroot_id), 580 "name": "Copr repository", 581 }] 582 583 if not copr.auto_createrepo: 584 repos.append({ 585 "id": "copr_base_devel", 586 "url": copr.repo_url + "/{}/devel/".format(chroot_id), 587 "name": "Copr buildroot", 588 }) 589 590 for repo in copr.repos_list: 591 repo_view = { 592 "id": generate_repo_name(repo), 593 "url": pre_process_repo_url(chroot_id, repo), 594 "name": "Additional repo " + generate_repo_name(repo), 595 } 596 repos.append(repo_view) 597 for repo in chroot.repos_list: 598 repo_view = { 599 "id": generate_repo_name(repo), 600 "url": pre_process_repo_url(chroot_id, repo), 601 "name": "Additional repo " + generate_repo_name(repo), 602 } 603 repos.append(repo_view) 604 605 return { 606 'project_id': copr.repo_id, 607 'additional_packages': packages.split(), 608 'repos': repos, 609 'chroot': chroot_id, 610 'use_bootstrap_container': copr.use_bootstrap_container 611 }
612