[mod] tools
Christian Fraß

Christian Fraß commited on 2021-03-03 00:26:17
Zeige 17 geänderte Dateien mit 55 Einfügungen und 382 Löschungen.

... ...
@@ -0,0 +1,4 @@
1
+#!/usr/bin/env bash
2
+
3
+make --file=tools/makefile
4
+
... ...
@@ -1,17 +0,0 @@
1
-import sys
2
-import json
3
-
4
-def main():
5
-	content = sys.stdin.read()
6
-	try:
7
-		data = json.loads(content)
8
-		print(content)
9
-		# print("valid")
10
-		return 0
11
-	except ValueError as error:
12
-		print("invalid")
13
-		print(error)
14
-		return -1
15
-
16
-sys.exit(main())
17
-
... ...
@@ -0,0 +1,4 @@
1
+#!/usr/bin/env bash
2
+
3
+rm -rf build
4
+
... ...
@@ -1,111 +0,0 @@
1
-import json as _json
2
-import os as _os
3
-import re as _re
4
-
5
-
6
-def dokuwiki_to(data):
7
-	content = ""
8
-	content += ("  * type: %s\n" % (data["type"]))
9
-	content += ("  * description: %s\n" % ("?" if (data["description"] is None) else data["description"]))
10
-	content += ("  * tags\n")
11
-	for tag in data["tags"]:
12
-		content += ("    * %s\n" % (tag))
13
-	content += ("  * translations\n")
14
-	for (language , lemmata) in data["translations"].items():
15
-		content += ("    * %s\n" % (language))
16
-		for lemma in lemmata:
17
-			content += ("      * %s\n" % (lemma))
18
-	return content
19
-	
20
-
21
-def dokuwiki_from(content):
22
-	data = {}
23
-	state = "type"
24
-	lines = content.split("\n")
25
-	for line in lines:
26
-		if (line != ""):
27
-			if (state == "type"):
28
-				result = _re.match("^  \\* type: (.*)$", line)
29
-				if (result is not None):
30
-					data["type"] = result.group(1)
31
-					state = "description"
32
-				else:
33
-					raise ValueError("type expected in line '%s' while being in state '%s'" % (line, state))
34
-			elif (state == "description"):
35
-				result = _re.match("^  \\* description: (.*)$", line)
36
-				if (result is not None):
37
-					data["description"] = (None if (result.group(1) == "?") else result.group(1))
38
-					state = "tags"
39
-				else:
40
-					raise ValueError("description expected in line '%s' while being in state '%s'" % (line, state))
41
-			elif (state == "tags"):
42
-				result = _re.match("^  \\* tags$", line)
43
-				if (result is not None):
44
-					data["tags"] = []
45
-					state = "tagentry"
46
-				else:
47
-					raise ValueError("tags expected in line '%s' while being in state '%s'" % (line, state))
48
-			elif (state == "tagentry"):
49
-				result = _re.match("^    \\* (.*)$", line)
50
-				if (result is not None):
51
-					data["tags"].append(result.group(1))
52
-					state = "tagentry"
53
-				else:
54
-					result = _re.match("^  \\* translations$", line)
55
-					if (result is not None):
56
-						data["translations"] = {}
57
-						state = "language"
58
-					else:
59
-						raise ValueError("tag or translations expected in line '%s' while being in state '%s'" % (line, state))
60
-			elif (state == "language"):
61
-				result = _re.match("^    \\* (.*)$", line)
62
-				if (result is not None):
63
-					language = result.group(1)
64
-					data["translations"][language] = []
65
-					state = "lemma"
66
-				else:
67
-					raise ValueError("language expected in line '%s' while being in state '%s'" % (line, state))
68
-			elif (state == "lemma"):
69
-				result = _re.match("^      \\* (.*)$", line)
70
-				if (result is not None):
71
-					lemma = result.group(1)
72
-					data["translations"][language].append(lemma)
73
-					state = "lemma"
74
-				else:
75
-					result = _re.match("^    \\* (.*)$", line)
76
-					if (result is not None):
77
-						language = result.group(1)
78
-						data["translations"][language] = []
79
-						state = "lemma"
80
-					else:
81
-						raise ValueError("language or lemma expected in line '%s' while being in state '%s'" % (line, state))
82
-			else:
83
-				raise ValueError("unhandled state '%s'" % (state))
84
-	return data
85
-	
86
-
87
-def json_to(data):
88
-	content = _json.dumps(data, indent = "\t", ensure_ascii = False, sort_keys = True)
89
-	return content
90
-	
91
-
92
-def json_from(content):
93
-	data = _json.loads(content)
94
-	return data
95
-	
96
-
97
-def filemap(folder_from, convert_from, folder_to, convert_to, namemapper):
98
-	for name_from in _os.listdir(folder_from):
99
-		path_from = _os.path.join(folder_from, name_from)
100
-		handle_from = open(path_from, "r")
101
-		content_from = handle_from.read()
102
-		handle_from.close()
103
-		data = convert_from(content_from)
104
-		name_to = namemapper(name_from)
105
-		path_to = _os.path.join(folder_to, name_to)
106
-		content_to = convert_to(data)
107
-		handle_to = open(path_to, "w")
108
-		handle_to.write(content_to)
109
-		handle_to.close()
110
-	
111
-
... ...
@@ -1,50 +0,0 @@
1
-#!/usr/bin/env python3
2
-
3
-import sys as _sys
4
-import os as _os
5
-import sqlite3 as _sqlite3
6
-import json as _json
7
-
8
-
9
-def main(args):
10
-	path = _os.path.join("queries", "convert.sql.tpl")
11
-	handle = open(path, "r")
12
-	querytemplate = handle.read()
13
-	handle.close()
14
-	arguments = {}
15
-	connection = _sqlite3.connect(_os.path.join("db", "words.sqlite"))
16
-	result = connection.cursor().execute(querytemplate, arguments).fetchall()
17
-	connection.close()
18
-	for index in range(len(result)):
19
-		line = result[index]
20
-		if True:
21
-			id_ = line[0]
22
-		if True:
23
-			type_ = line[1]
24
-		if True:
25
-			description = line[2]
26
-		if True:
27
-			tags = ([] if (line[3] is None) else line[3].split("|"))
28
-		if True:
29
-			translations = {}
30
-			pairs = list(map(lambda x: tuple(x.split(":", 2)), line[4].split("|")))
31
-			for (language , lemma) in sorted(pairs, key = lambda pair: pair[0]):
32
-				if (language not in translations):
33
-					translations[language] = []
34
-				translations[language].append(lemma)
35
-		obj = {
36
-			"type": type_,
37
-			"description": description,
38
-			"tags": tags,
39
-			"translations": translations,
40
-		}
41
-		name = ("concept_%u.json" % (id_))
42
-		path = _os.path.join("concepts", name)
43
-		output = _json.dumps(obj, indent = "\t", ensure_ascii = False)
44
-		handle = open(path, "w")
45
-		handle.write(output)
46
-		handle.close()
47
-	
48
-
49
-main(_sys.argv[1:])
50
-
... ...
@@ -1,33 +0,0 @@
1
-import sys
2
-import json
3
-
4
-def stringify(x):
5
-	if (x is None):
6
-		return "null"
7
-	elif (type(x) == str):
8
-		return ("\"%s\"" % x)
9
-	elif (type(x) == list):
10
-		return ("[%s]" % (",".join(map(lambda y: stringify(y), x))))
11
-	else:
12
-		return str(x)
13
-
14
-def main():
15
-	languages = ["fs","en","af","nl","yi","de","da","nb","sv","nn","is","pg"]
16
-	data = json.loads(sys.stdin.read())
17
-	sys.stdout.write("[\n")
18
-	first = True
19
-	for entry in data:
20
-		if (first):
21
-			first = False
22
-		else:
23
-			sys.stdout.write(",\n")
24
-		s = ""
25
-		s += "\t{\n"
26
-		s += "\t\t%s: %s, %s: %s, %s: %s,\n" % (stringify("type"), stringify(entry["type"]), stringify("description"), stringify(entry["description"]), stringify("tags"), stringify(entry["tags"]))
27
-		s += "\t\t%s: {%s}\n" % (stringify("translations"), ", ".join(map(lambda language: "%s: %s" % (stringify(language), stringify(entry["translations"][language])), languages)))
28
-		s += "\t}"
29
-		sys.stdout.write(s)
30
-	sys.stdout.write("\n]\n")
31
-
32
-main()
33
-
... ...
@@ -0,0 +1,47 @@
1
+## directories
2
+dir_source := source
3
+dir_build := build
4
+dir_lib := lib
5
+
6
+## commands
7
+cmd_dir_make := mkdir --parents
8
+cmd_copy := cp --recursive --update --verbose
9
+cmd_tsc := tsc --lib es2015,dom --target es6
10
+cmd_remove := rm --force
11
+cmd_link := ln --symbolic
12
+
13
+
14
+all: ${dir_build}/sql ${dir_build}/manage
15
+.PHONY: all
16
+
17
+${dir_build}/sql:
18
+	@ ${cmd_dir_make} ${dir_build}/sql
19
+	@ ${cmd_copy} ${dir_source}/sql/* ${dir_build}/sql/
20
+.PHONY: ${dir_build}/sql
21
+
22
+${dir_build}/manage: \
23
+	${dir_source}/types.ts \
24
+	${dir_source}/helpers/string.ts \
25
+	${dir_source}/helpers/file.ts \
26
+	${dir_source}/helpers/database.ts \
27
+	${dir_source}/helpers/misc.ts \
28
+	${dir_source}/helpers/module.ts \
29
+	${dir_source}/helpers/storage.ts \
30
+	${dir_source}/helpers/repository.ts \
31
+	${dir_source}/entities/concept.ts \
32
+	${dir_source}/repositories/language.ts \
33
+	${dir_source}/repositories/type.ts \
34
+	${dir_source}/repositories/tag.ts \
35
+	${dir_source}/repositories/concept-core.ts \
36
+	${dir_source}/repositories/concept-tags.ts \
37
+	${dir_source}/repositories/concept-translations.ts \
38
+	${dir_source}/repositories/concept.ts \
39
+	${dir_source}/services/language.ts \
40
+	${dir_source}/services/type.ts \
41
+	${dir_source}/services/tag.ts \
42
+	${dir_source}/services/concept.ts \
43
+	${dir_source}/main.ts
44
+	@ ${cmd_dir_make} ${dir_build}
45
+	@ ${cmd_tsc} $^ --outFile $@
46
+	@ chmod +x ${dir_build}/manage
47
+
... ...
@@ -1,21 +0,0 @@
1
-#!/usr/bin/env python3
2
-
3
-import os as _os
4
-import json as _json
5
-
6
-
7
-def main():
8
-	data = []
9
-	folder = "concepts/json"
10
-	for name in sorted(_os.listdir(folder)):
11
-		path = _os.path.join(folder, name)
12
-		handle = open(path, "r")
13
-		content = handle.read()
14
-		handle.close()
15
-		entry = _json.loads(content)
16
-		data.append(entry)
17
-	print(_json.dumps(data))
18
-	
19
-
20
-main()
21
-
... ...
@@ -1,17 +0,0 @@
1
-#!/usr/bin/env sh
2
-
3
-path=concepts.json
4
-python3 tools/merge.py > ${path}
5
-
6
-destination=mehl:/home/bfadmin/websites/folksprak.org/htdocs/data
7
-rsync \
8
-	--verbose \
9
-	--archive \
10
-	--compress \
11
-	--recursive \
12
-	--rsh=ssh \
13
-	${path} \
14
-	${destination}/
15
-
16
-rm ${path}
17
-
... ...
@@ -1,23 +0,0 @@
1
-import sys
2
-import json
3
-
4
-def index(type_):
5
-	return {
6
-		"verb": 00,
7
-		"attribute": 10,
8
-		"noun": 20,
9
-		"pronoun": 30,
10
-		"correlative": 40,
11
-		"preposition": 50,
12
-		"article": 60,
13
-		"number": 70,
14
-		"other": 80,
15
-	}[type_]
16
-
17
-def main():
18
-	data = json.loads(sys.stdin.read())
19
-	data_ = sorted(data, key = lambda entry: index(entry["type"]))
20
-	print(json.dumps(data_, indent="\t"))
21
-
22
-main()
23
-
... ...
@@ -1,35 +0,0 @@
1
-import sys
2
-import json
3
-
4
-def sqlformat(x):
5
-	if (x is None):
6
-		return "NULL"
7
-	elif (type(x) == str):
8
-		return ("'%s'" % x.replace("'", "\\'"))
9
-	else:
10
-		return str(x)
11
-
12
-def lastid(tablename):
13
-	return ("(SELECT id FROM %s ORDER BY id DESC LIMIT 1)" % (tablename))
14
-
15
-def main():
16
-	data = json.loads(sys.stdin.read())
17
-	print(len(data))
18
-	return
19
-	for entry in data[324:]:
20
-		query = ""
21
-		query += ("INSERT INTO\n\tconcepts(type,description)\nVALUES\n\t(%s,%s)\n;\n" % (sqlformat(entry["type"]), sqlformat(entry["description"])))
22
-		concept_id = lastid("concepts")
23
-		for (iso_639_2,lemmata) in list(entry["translations"].items()):
24
-			for lemma in lemmata:
25
-				query += ("INSERT INTO\n\twords(language_id,lemma,pronunciation)\nVALUES\n\t((SELECT id FROM languages WHERE iso_639_2=%s),%s,%s)\n;\n" % (sqlformat(iso_639_2), sqlformat(lemma), sqlformat(None)))
26
-				word_id = lastid("words")
27
-				strength = None
28
-				query += ("INSERT INTO\n\tcorrelations(concept_id,word_id,strength)\nVALUES\n\t(%s,%s,%s)\n;\n" % (concept_id, word_id, sqlformat(strength)))
29
-		for tag in entry["tags"]:
30
-			query += ("INSERT INTO tags(concept_id,tag) VALUES (%s,%s);\n" % (concept_id, sqlformat(tag)))
31
-		query += "--\n"
32
-		sys.stdout.write(query)
33
-
34
-main()
35
-
... ...
@@ -1,19 +0,0 @@
1
-#!/usr/bin/env python3
2
-
3
-import sys as _sys
4
-
5
-import common as _common
6
-
7
-
8
-def main(args):
9
-	_common.filemap(
10
-		"concepts/json",
11
-		_common.json_from,
12
-		"concepts/wiki",
13
-		_common.dokuwiki_to,
14
-		lambda name_from: ("concept_%u.txt" % (int(name_from.split(".")[0].split("_")[1])))
15
-	)
16
-	
17
-
18
-main(_sys.argv[1:])
19
-
... ...
@@ -1,14 +0,0 @@
1
-#!/usr/bin/env sh
2
-
3
-source=mehl:/home/bfadmin/websites/folksprak.org/htdocs/wiki/data/pages/concepts
4
-destination=concepts/wiki
5
-mkdir --parents ${destination}
6
-rsync \
7
-	--verbose \
8
-	--archive \
9
-	--compress \
10
-	--recursive \
11
-	--rsh=ssh \
12
-	${source}/* \
13
-	${destination}/
14
-
... ...
@@ -1,13 +0,0 @@
1
-#!/usr/bin/env sh
2
-
3
-source=concepts/wiki
4
-destination=mehl:/home/bfadmin/websites/folksprak.org/htdocs/wiki/data/pages/concepts
5
-rsync \
6
-	--verbose \
7
-	--archive \
8
-	--compress \
9
-	--recursive \
10
-	--rsh=ssh \
11
-	${source}/* \
12
-	${destination}/
13
-
... ...
@@ -1,5 +0,0 @@
1
-#!/usr/bin/env sh
2
-
3
-tools/wiki-in.sh
4
-python3 tools/wiki-take.py
5
-
... ...
@@ -1,5 +0,0 @@
1
-#!/usr/bin/env sh
2
-
3
-python3 tools/wiki-give.py
4
-tools/wiki-out.sh
5
-
... ...
@@ -1,19 +0,0 @@
1
-#!/usr/bin/env python3
2
-
3
-import sys as _sys
4
-
5
-import common as _common
6
-
7
-
8
-def main(args):
9
-	_common.filemap(
10
-		"concepts/wiki",
11
-		_common.dokuwiki_from,
12
-		"concepts/json",
13
-		_common.json_to,
14
-		lambda name_from: ("concept_%u.json" % (int(name_from.split(".")[0].split("_")[1])))
15
-	)
16
-	
17
-
18
-main(_sys.argv[1:])
19
-
20 0