summaryrefslogtreecommitdiffstats
path: root/support/schema/gendummydata.py
diff options
context:
space:
mode:
authorLukas Fleischer <archlinux@cryptocrack.de>2014-02-06 19:09:15 +0100
committerLukas Fleischer <archlinux@cryptocrack.de>2014-02-06 19:11:25 +0100
commitb7b586a8c405e6e5967c5febcab67caab581ca0c (patch)
treefcdaf036a036b7e554b07734d7ce32918b67c192 /support/schema/gendummydata.py
parentd0c927c940435bad3da922f727713e1b06b6c936 (diff)
downloadaurweb-b7b586a8c405e6e5967c5febcab67caab581ca0c.tar.xz
Move support/schema/ to schema/
There aren't any other subdirectories in support/. Reduce the nesting depth by moving schema/ to the top-level source directory. Signed-off-by: Lukas Fleischer <archlinux@cryptocrack.de>
Diffstat (limited to 'support/schema/gendummydata.py')
-rwxr-xr-xsupport/schema/gendummydata.py302
1 files changed, 0 insertions, 302 deletions
diff --git a/support/schema/gendummydata.py b/support/schema/gendummydata.py
deleted file mode 100755
index 361d1f9..0000000
--- a/support/schema/gendummydata.py
+++ /dev/null
@@ -1,302 +0,0 @@
-#!/usr/bin/python3
-"""
-usage: gendummydata.py outputfilename.sql
-"""
-#
-# This script seeds the AUR database with dummy data for
-# use during development/testing. It uses random entries
-# from /usr/share/dict/words to create user accounts and
-# package names. It generates the SQL statements to
-# insert these users/packages into the AUR database.
-#
-import random
-import time
-import os
-import sys
-import io
-import logging
-
-LOG_LEVEL = logging.DEBUG # logging level. set to logging.INFO to reduce output
-SEED_FILE = "/usr/share/dict/words"
-DB_HOST = os.getenv("DB_HOST", "localhost")
-DB_NAME = os.getenv("DB_NAME", "AUR")
-DB_USER = os.getenv("DB_USER", "aur")
-DB_PASS = os.getenv("DB_PASS", "aur")
-USER_ID = 5 # Users.ID of first bogus user
-PKG_ID = 1 # Packages.ID of first package
-MAX_USERS = 300 # how many users to 'register'
-MAX_DEVS = .1 # what percentage of MAX_USERS are Developers
-MAX_TUS = .2 # what percentage of MAX_USERS are Trusted Users
-MAX_PKGS = 900 # how many packages to load
-PKG_DEPS = (1, 5) # min/max depends a package has
-PKG_SRC = (1, 3) # min/max sources a package has
-PKG_CMNTS = (1, 5) # min/max number of comments a package has
-CATEGORIES_COUNT = 17 # the number of categories from aur-schema
-VOTING = (0, .30) # percentage range for package voting
-OPEN_PROPOSALS = 5 # number of open trusted user proposals
-CLOSE_PROPOSALS = 15 # number of closed trusted user proposals
-RANDOM_TLDS = ("edu", "com", "org", "net", "tw", "ru", "pl", "de", "es")
-RANDOM_URL = ("http://www.", "ftp://ftp.", "http://", "ftp://")
-RANDOM_LOCS = ("pub", "release", "files", "downloads", "src")
-FORTUNE_FILE = "/usr/share/fortune/cookie"
-
-# setup logging
-logformat = "%(levelname)s: %(message)s"
-logging.basicConfig(format=logformat, level=LOG_LEVEL)
-log = logging.getLogger()
-
-if len(sys.argv) != 2:
- log.error("Missing output filename argument")
- raise SystemExit
-
-# make sure the seed file exists
-#
-if not os.path.exists(SEED_FILE):
- log.error("Please install the 'words' Arch package")
- raise SystemExit
-
-# make sure comments can be created
-#
-if not os.path.exists(FORTUNE_FILE):
- log.error("Please install the 'fortune-mod' Arch package")
- raise SystemExit
-
-# track what users/package names have been used
-#
-seen_users = {}
-seen_pkgs = {}
-user_keys = []
-
-# some functions to generate random data
-#
-def genVersion():
- ver = []
- ver.append("%d" % random.randrange(0,10))
- ver.append("%d" % random.randrange(0,20))
- if random.randrange(0,2) == 0:
- ver.append("%d" % random.randrange(0,100))
- return ".".join(ver) + "-%d" % random.randrange(1,11)
-def genCategory():
- return random.randrange(1,CATEGORIES_COUNT)
-def genUID():
- return seen_users[user_keys[random.randrange(0,len(user_keys))]]
-def genFortune():
- return fortunes[random.randrange(0,len(fortunes))].replace("'", "")
-
-
-# load the words, and make sure there are enough words for users/pkgs
-#
-log.debug("Grabbing words from seed file...")
-fp = open(SEED_FILE, "r", encoding="utf-8")
-contents = fp.readlines()
-fp.close()
-if MAX_USERS > len(contents):
- MAX_USERS = len(contents)
-if MAX_PKGS > len(contents):
- MAX_PKGS = len(contents)
-if len(contents) - MAX_USERS > MAX_PKGS:
- need_dupes = 0
-else:
- need_dupes = 1
-
-# select random usernames
-#
-log.debug("Generating random user names...")
-user_id = USER_ID
-while len(seen_users) < MAX_USERS:
- user = random.randrange(0, len(contents))
- word = contents[user].replace("'", "").replace(".","").replace(" ", "_")
- word = word.strip().lower()
- if word not in seen_users:
- seen_users[word] = user_id
- user_id += 1
-user_keys = list(seen_users.keys())
-
-# select random package names
-#
-log.debug("Generating random package names...")
-num_pkgs = PKG_ID
-while len(seen_pkgs) < MAX_PKGS:
- pkg = random.randrange(0, len(contents))
- word = contents[pkg].replace("'", "").replace(".","").replace(" ", "_")
- word = word.strip().lower()
- if not need_dupes:
- if word not in seen_pkgs and word not in seen_users:
- seen_pkgs[word] = num_pkgs
- num_pkgs += 1
- else:
- if word not in seen_pkgs:
- seen_pkgs[word] = num_pkgs
- num_pkgs += 1
-
-# free up contents memory
-#
-contents = None
-
-# developer/tu IDs
-#
-developers = []
-trustedusers = []
-has_devs = 0
-has_tus = 0
-
-# Just let python throw the errors if any happen
-#
-out = open(sys.argv[1], "w", encoding="utf-8")
-out.write("BEGIN;\n")
-
-# Begin by creating the User statements
-#
-log.debug("Creating SQL statements for users.")
-for u in user_keys:
- account_type = 1 # default to normal user
- if not has_devs or not has_tus:
- account_type = random.randrange(1, 4)
- if account_type == 3 and not has_devs:
- # this will be a dev account
- #
- developers.append(seen_users[u])
- if len(developers) >= MAX_DEVS * MAX_USERS:
- has_devs = 1
- elif account_type == 2 and not has_tus:
- # this will be a trusted user account
- #
- trustedusers.append(seen_users[u])
- if len(trustedusers) >= MAX_TUS * MAX_USERS:
- has_tus = 1
- else:
- # a normal user account
- #
- pass
-
- s = ("INSERT INTO Users (ID, AccountTypeID, Username, Email, Passwd)"
- " VALUES (%d, %d, '%s', '%s@example.com', MD5('%s'));\n")
- s = s % (seen_users[u], account_type, u, u, u)
- out.write(s)
-
-log.debug("Number of developers: %d" % len(developers))
-log.debug("Number of trusted users: %d" % len(trustedusers))
-log.debug("Number of users: %d" % (MAX_USERS-len(developers)-len(trustedusers)))
-log.debug("Number of packages: %d" % MAX_PKGS)
-
-log.debug("Gathering text from fortune file...")
-fp = open(FORTUNE_FILE, "r", encoding="utf-8")
-fortunes = fp.read().split("%\n")
-fp.close()
-
-# Create the package statements
-#
-log.debug("Creating SQL statements for packages.")
-count = 0
-for p in list(seen_pkgs.keys()):
- NOW = int(time.time())
- if count % 2 == 0:
- muid = developers[random.randrange(0,len(developers))]
- else:
- muid = trustedusers[random.randrange(0,len(trustedusers))]
- if count % 20 == 0: # every so often, there are orphans...
- muid = "NULL"
-
- uuid = genUID() # the submitter/user
-
- s = ("INSERT INTO Packages (ID, Name, Version, CategoryID,"
- " SubmittedTS, SubmitterUID, MaintainerUID) VALUES "
- " (%d, '%s', '%s', %d, %d, %d, %s);\n")
- s = s % (seen_pkgs[p], p, genVersion(), genCategory(), NOW, uuid, muid)
-
- out.write(s)
- count += 1
-
- # create random comments for this package
- #
- num_comments = random.randrange(PKG_CMNTS[0], PKG_CMNTS[1])
- for i in range(0, num_comments):
- now = NOW + random.randrange(400, 86400*3)
- s = ("INSERT INTO PackageComments (PackageID, UsersID,"
- " Comments, CommentTS) VALUES (%d, %d, '%s', %d);\n")
- s = s % (seen_pkgs[p], genUID(), genFortune(), now)
- out.write(s)
-
-# Cast votes
-#
-track_votes = {}
-log.debug("Casting votes for packages.")
-for u in user_keys:
- num_votes = random.randrange(int(len(seen_pkgs)*VOTING[0]),
- int(len(seen_pkgs)*VOTING[1]))
- pkgvote = {}
- for v in range(num_votes):
- pkg = random.randrange(1, len(seen_pkgs) + 1)
- if pkg not in pkgvote:
- s = ("INSERT INTO PackageVotes (UsersID, PackageID)"
- " VALUES (%d, %d);\n")
- s = s % (seen_users[u], pkg)
- pkgvote[pkg] = 1
- if pkg not in track_votes:
- track_votes[pkg] = 0
- track_votes[pkg] += 1
- out.write(s)
-
-# Update statements for package votes
-#
-for p in list(track_votes.keys()):
- s = "UPDATE Packages SET NumVotes = %d WHERE ID = %d;\n"
- s = s % (track_votes[p], p)
- out.write(s)
-
-# Create package dependencies and sources
-#
-log.debug("Creating statements for package depends/sources.")
-for p in list(seen_pkgs.keys()):
- num_deps = random.randrange(PKG_DEPS[0], PKG_DEPS[1])
- this_deps = {}
- i = 0
- while i != num_deps:
- dep = random.choice([k for k in seen_pkgs])
- if dep not in this_deps:
- s = "INSERT INTO PackageDepends VALUES (%d, '%s', NULL);\n"
- s = s % (seen_pkgs[p], dep)
- out.write(s)
- i += 1
-
- num_sources = random.randrange(PKG_SRC[0], PKG_SRC[1])
- for i in range(num_sources):
- src_file = user_keys[random.randrange(0, len(user_keys))]
- src = "%s%s.%s/%s/%s-%s.tar.gz" % (
- RANDOM_URL[random.randrange(0,len(RANDOM_URL))],
- p, RANDOM_TLDS[random.randrange(0,len(RANDOM_TLDS))],
- RANDOM_LOCS[random.randrange(0,len(RANDOM_LOCS))],
- src_file, genVersion())
- s = "INSERT INTO PackageSources VALUES (%d, '%s');\n"
- s = s % (seen_pkgs[p], src)
- out.write(s)
-
-# Create trusted user proposals
-#
-log.debug("Creating SQL statements for trusted user proposals.")
-count=0
-for t in range(0, OPEN_PROPOSALS+CLOSE_PROPOSALS):
- now = int(time.time())
- if count < CLOSE_PROPOSALS:
- start = now - random.randrange(3600*24*7, 3600*24*21)
- end = now - random.randrange(0, 3600*24*7)
- else:
- start = now
- end = now + random.randrange(3600*24, 3600*24*7)
- if count % 5 == 0: # Don't make the vote about anyone once in a while
- user = ""
- else:
- user = user_keys[random.randrange(0,len(user_keys))]
- suid = trustedusers[random.randrange(0,len(trustedusers))]
- s = ("INSERT INTO TU_VoteInfo (Agenda, User, Submitted, End,"
- " SubmitterID) VALUES ('%s', '%s', %d, %d, %d);\n")
- s = s % (genFortune(), user, start, end, suid)
- out.write(s)
- count += 1
-
-# close output file
-#
-out.write("COMMIT;\n")
-out.write("\n")
-out.close()
-log.debug("Done.")