diff --git a/docs/_config.yml b/docs/_config.yml
index c4a31711..87e1ba9c 100644
--- a/docs/_config.yml
+++ b/docs/_config.yml
@@ -1,2 +1,2 @@
-dataURL: https://autodesk.github.io/hubble/demo-data
+dataURL: https://steffen.github.io/hubble/demo-data
exclude: ['spec', 'node_modules', 'README.md', '.gitignore', 'package.json', 'package-lock.json']
diff --git a/docs/_data/navigation.yml b/docs/_data/navigation.yml
index ffa9bb07..56ba6e9d 100644
--- a/docs/_data/navigation.yml
+++ b/docs/_data/navigation.yml
@@ -25,6 +25,8 @@
url: "/repos-activity"
- title: "Total"
url: "/repos-total"
+ - title: "Monitored"
+ url: "/repos-monitored"
- title: "Organizations"
url: "/orgs-activity"
subnavigation:
diff --git a/docs/_data/repository_navigation.yml b/docs/_data/repository_navigation.yml
new file mode 100644
index 00000000..8e41e4de
--- /dev/null
+++ b/docs/_data/repository_navigation.yml
@@ -0,0 +1,14 @@
+- title: "Overview"
+ url: "/repository/"
+- title: "Git Metrics"
+ url: "/repository/git-repository-size"
+ subnavigation:
+ - title: "Repository Size"
+ url: "/repository/git-repository-size"
+- title: "GitHub Metrics"
+ url: "/repository/github-git-traffic"
+ subnavigation:
+ - title: "Git Traffic"
+ url: "/repository/github-git-traffic"
+ - title: "API Requests"
+ url: "/repository/github-api-requests"
diff --git a/docs/_includes/repository_navigation.html b/docs/_includes/repository_navigation.html
new file mode 100644
index 00000000..3845ad16
--- /dev/null
+++ b/docs/_includes/repository_navigation.html
@@ -0,0 +1,45 @@
+{% include environment.html %}
+
+ {% if site.github.repository_nwo != nil %}
+ {% assign site.baseurl = site.github.repository_nwo %}
+ {% endif %}
+ {% assign activeItem = nil %}
+ {% assign activeSubitem = nil %}
+ {% for item in site.data.repository_navigation %}
+ {% if item.url == page.url %}
+ {% assign activeItem = item %}
+ {% endif %}
+ {% for subitem in item.subnavigation %}
+ {% if subitem.url == page.url %}
+ {% assign activeItem = item %}
+ {% assign activeSubitem = subitem %}
+ {% endif %}
+ {% endfor %}
+ {% endfor %}
+
+
+ {% for item in site.data.repository_navigation %}
+
+ {% if item == activeItem %}
+ {{ item.title }}
+ {% else %}
+ {{ item.title }}
+ {% endif %}
+
+ {% endfor %}
+
+ {% if activeItem != nil and activeItem.subnavigation != nil %}
+
+ {% endif %}
+
+
diff --git a/docs/_layouts/repository.html b/docs/_layouts/repository.html
new file mode 100644
index 00000000..49551307
--- /dev/null
+++ b/docs/_layouts/repository.html
@@ -0,0 +1,61 @@
+{% include environment.html %}
+
+
+
+
+ Hubble Enterprise{% if page.title %}: {{ page.title }}{% endif %}
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ {% include repository_navigation.html %}
+
+ {{ content }}
+ {% include footer.html %}
+
+
+
diff --git a/docs/assets/css/repository.css b/docs/assets/css/repository.css
new file mode 100644
index 00000000..9e83697d
--- /dev/null
+++ b/docs/assets/css/repository.css
@@ -0,0 +1,19 @@
+.page-header {
+ background-color: #275d9c;
+}
+
+.back-link {
+ position: absolute;
+ text-transform: uppercase;
+ font-size: 12px;
+ padding: 1.2rem 1.5rem;
+ text-decoration: none;
+}
+
+.back-link a {
+ text-decoration: none;
+}
+
+.back-link a:hover {
+ text-decoration: underline;
+}
\ No newline at end of file
diff --git a/docs/assets/js/_partials/chart-table.js b/docs/assets/js/_partials/chart-table.js
index 6de3d001..a0728a26 100644
--- a/docs/assets/js/_partials/chart-table.js
+++ b/docs/assets/js/_partials/chart-table.js
@@ -1,6 +1,7 @@
function createTable(table)
{
const url = $(table).data('url');
+ const noThead = $(table).data('no-thead');
let spinner = createSpinner(table);
@@ -13,14 +14,15 @@ function createTable(table)
if (data.length == 0)
return;
- d3.select(table)
- .append('thead')
- .append('tr')
- .selectAll('th')
- .data(data.columns)
- .enter()
- .append('th')
- .text(d => d);
+ if (noThead === undefined)
+ d3.select(table)
+ .append('thead')
+ .append('tr')
+ .selectAll('th')
+ .data(data.columns)
+ .enter()
+ .append('th')
+ .text(d => d);
let displayData = data;
const configSlice = readConfig($(table), 'slice');
@@ -61,9 +63,12 @@ function createTable(table)
case 'repository':
case 'resource':
case 'user':
+ const tableLinkHref = d3.select(table).attr('data-link-href');
+ const tableLinkTarget = d3.select(table).attr('data-link-target');
+
let a = cell.append('a').text(entry)
- .attr('target', '_blank')
- .attr('href', gheUrl() + '/' + entry)
+ .attr('target', tableLinkTarget || '_blank')
+ .attr('href', tableLinkHref ? tableLinkHref + entry : gheUrl() + '/' + entry)
.text(entry);
const tableID = d3.select(table).attr('id');
diff --git a/docs/assets/js/repository.js b/docs/assets/js/repository.js
new file mode 100644
index 00000000..b453c2fa
--- /dev/null
+++ b/docs/assets/js/repository.js
@@ -0,0 +1,23 @@
+$(window).bind('load', function()
+{
+ const queryString = new URLSearchParams(location.search);
+ const nwo = queryString.get('nwo');
+
+ // Add nwo query string to all navigation links
+ $('.navigation a').each(function()
+ {
+ const href = $(this).attr('href');
+ const noNwoQuery = $(this).attr('data-no-nwo-query')
+
+ if (noNwoQuery === undefined) {
+ $(this).attr('href', `${href}?nwo=${nwo}`);
+ }
+ });
+
+ // Replace all canvas data urls with repository specific urls
+ $('[data-url]').each(function()
+ {
+ const dataUrl = $(this).attr('data-url');
+ $(this).attr('data-url', dataUrl.replace('/repository', `/repository/${nwo}`));
+ });
+});
\ No newline at end of file
diff --git a/docs/demo-data/repository-monitored.tsv b/docs/demo-data/repository-monitored.tsv
new file mode 100644
index 00000000..abeb98ad
--- /dev/null
+++ b/docs/demo-data/repository-monitored.tsv
@@ -0,0 +1,2 @@
+repository
+test-org/large-repo
diff --git a/docs/demo-data/repository/test-org/large-repo/git-sizer.tsv b/docs/demo-data/repository/test-org/large-repo/git-sizer.tsv
new file mode 100644
index 00000000..91b5fd09
--- /dev/null
+++ b/docs/demo-data/repository/test-org/large-repo/git-sizer.tsv
@@ -0,0 +1,6 @@
+date commits commits [MiB] trees trees [MiB] tree entries blobs blobs [MiB] annotated tags refs biggest commit [KiB] most parents most tree entries biggest blob [MiB] largest history depth largest tag depth directories path depth path length files size of files [MiB] symlinks submodules
+2019-01-01 800564 585.22 3732337 9638.486 253108785 1330280 60880.353 53 341 72.7 66 1895 13.496 145316 0 4434 13 134 63153 774.755 40 3
+2019-01-02 803564 589.22 3752337 9838.486 272108785 1420280 61280.353 53 360 72.7 66 1895 13.496 145316 0 4434 13 134 63153 774.755 40 3
+2019-01-03 806564 595.22 3772337 9938.486 285108785 1550280 62480.353 54 370 72.7 66 1895 13.496 145316 0 4434 13 134 63153 774.755 40 3
+2019-01-04 809564 500.22 3792337 10038.486 290108785 1770280 63380.353 55 350 72.7 66 1895 13.496 145316 0 4434 13 134 63153 774.755 40 4
+2019-01-05 811564 605.22 3832337 10638.486 303108785 1830280 64880.353 55 360 72.7 66 1895 13.496 145316 0 4434 13 134 63153 774.755 40 4
diff --git a/docs/demo-data/repository/test-org/large-repo/github-api-request-types-by-count.tsv b/docs/demo-data/repository/test-org/large-repo/github-api-request-types-by-count.tsv
new file mode 100644
index 00000000..7f47d9dc
--- /dev/null
+++ b/docs/demo-data/repository/test-org/large-repo/github-api-request-types-by-count.tsv
@@ -0,0 +1,11 @@
+type count
+"/repositories/:repository_id/issues/:issue_number/comments" 10620
+"/repositories/:repository_id/issues/:issue_number" 9624
+"/repositories/:repository_id/pulls/:pull_number" 9559
+"/repositories/:repository_id/pulls" 1599
+"/repositories/:repository_id" 291
+"/repositories/:repository_id/contents/?*" 220
+ "/repositories/:repository_id/branches" 130
+ "/repositories/:repository_id/commits" 122
+ "/repositories/:repository_id/languages" 120
+ "/repositories/:repository_id/commits/*" 119
\ No newline at end of file
diff --git a/docs/demo-data/repository/test-org/large-repo/github-git-download-detailed.tsv b/docs/demo-data/repository/test-org/large-repo/github-git-download-detailed.tsv
new file mode 100644
index 00000000..23b8984e
--- /dev/null
+++ b/docs/demo-data/repository/test-org/large-repo/github-git-download-detailed.tsv
@@ -0,0 +1,26 @@
+repository user cloning? requests download [GB]
+test-org/large-repo someUser5322 true 7 37
+test-org/large-repo someUser5337 true 44 36
+test-org/large-repo someUser2762 true 40 34
+test-org/large-repo someUser8267 true 27 28
+test-org/large-repo someUser1992 true 35 26
+test-org/large-repo someUser2706 true 110 23
+test-org/large-repo someUser5302 true 27 22
+test-org/large-repo someUser5357 true 80 21
+test-org/large-repo someUser5317 true 139 18
+test-org/large-repo someUser4337 true 76 16
+test-org/large-repo someUser5137 true 74 11
+test-org/large-repo someUser3589 true 10 10
+test-org/large-repo someUser5134 true 33 9
+test-org/large-repo someUser1528 true 63 9
+test-org/large-repo someUser5737 true 43 7
+test-org/large-repo someUser2064 true 6 6
+test-org/large-repo someUser2736 true 40 6
+test-org/large-repo someUser5550 true 7 6
+test-org/large-repo someUser5347 true 129 6
+test-org/large-repo someUser2434 true 20 6
+test-org/large-repo someUser3526 true 21 6
+test-org/large-repo someUser4112 true 62 6
+test-org/large-repo someUser5307 false 29 5
+test-org/large-repo someUser1322 true 68 5
+test-org/large-repo someUser1972 true 33 5
diff --git a/docs/demo-data/repository/test-org/large-repo/github-git-download.tsv b/docs/demo-data/repository/test-org/large-repo/github-git-download.tsv
new file mode 100644
index 00000000..225fe0c1
--- /dev/null
+++ b/docs/demo-data/repository/test-org/large-repo/github-git-download.tsv
@@ -0,0 +1,19 @@
+date clones fetches clone traffic [GB] fetch traffic [GB] LFS traffic [GB]
+2017-10-04 1722 2248 67 5 48
+2017-10-03 1867 2123 57 8 40
+2017-10-02 1605 2744 52 2 48
+2017-10-01 608 359 18 2 8
+2017-09-30 509 224 17 0 8.8
+2017-09-29 1962 2087 65 2 45
+2017-09-28 2352 2783 62 7 48
+2017-09-26 2113 2656 73 4 44
+2017-09-25 1840 2967 60 7 40
+2017-09-24 798 676 27 9 8.6
+2017-09-23 517 242 16 3 8
+2017-09-22 1498 2414 56 3 44
+2017-09-21 2035 2739 76 5 41
+2017-09-20 1719 2829 87 9 46
+2017-09-19 1864 2432 84 5 48
+2017-09-18 1995 2742 89 6 46
+2017-09-17 633 536 15 2 17
+2017-09-16 526 263 13 1 8.2
diff --git a/docs/demo-data/repository/test-org/large-repo/overview.tsv b/docs/demo-data/repository/test-org/large-repo/overview.tsv
new file mode 100644
index 00000000..bb6ef52e
--- /dev/null
+++ b/docs/demo-data/repository/test-org/large-repo/overview.tsv
@@ -0,0 +1,3 @@
+key value
+Repository Name test-org/large-repo
+Repository Size 2340 MB
diff --git a/docs/repos-monitored.html b/docs/repos-monitored.html
new file mode 100644
index 00000000..622b1c28
--- /dev/null
+++ b/docs/repos-monitored.html
@@ -0,0 +1,16 @@
+---
+layout: default
+title: Monitored Repository
+permalink: /repos-monitored
+---
+
+
+
Monitored Repositories
+
+
diff --git a/docs/repository/git-repository-size.html b/docs/repository/git-repository-size.html
new file mode 100644
index 00000000..ce36fe29
--- /dev/null
+++ b/docs/repository/git-repository-size.html
@@ -0,0 +1,279 @@
+---
+layout: repository
+title: Git Metrics - Repository Size
+permalink: /repository/git-repository-size
+---
+
+
+
Number of Commits and Depth of History
+
+
+
+ The total number of commits and the maximum history depth. Cloning a repository with a large number of commits and a deep history and running certain local Git commands on it (e.g. git blame) can be slow.
+
+
+
+
+
+
Size of Blobs
+
+
+
+ The total size of blobs in MiB. Avoid a rapid growth of the total size of blobs by using LFS for larger files.
+
+
+
+
+
+
Number of Refs
+
+
+
+ The total number of branches and tags. Cloning a repository with a large number of branches and tags can take a lot of time and local Git operations (e.g. tab-complete a branch name) can be slow.
+
+
+
+
+
+
Number of Submodules
+
+
+
+ The total number of submodules. Git needs to query their status on all kinds of local operations and this query operation is in particular slow on Windows.
+
+
+
\ No newline at end of file
diff --git a/docs/repository/github-api-requests.html b/docs/repository/github-api-requests.html
new file mode 100644
index 00000000..14600cfb
--- /dev/null
+++ b/docs/repository/github-api-requests.html
@@ -0,0 +1,19 @@
+---
+layout: repository
+title: GitHub Metrics - API Requests
+permalink: /repository/github-api-requests
+---
+
+
+
API Request Types by Count
+
+
+
+ The top 20 api request types by count in the last 24h.
+
+
+
diff --git a/docs/repository/github-git-traffic.html b/docs/repository/github-git-traffic.html
new file mode 100644
index 00000000..f3371ed9
--- /dev/null
+++ b/docs/repository/github-git-traffic.html
@@ -0,0 +1,141 @@
+---
+layout: repository
+title: GitHub Metrics - Git Traffic
+permalink: /repository/github-git-traffic
+---
+
+
+
Operations
+
+
+
+ Cloning a Git repository is expensive, as the entire repository history must be transferred.
+ Clients that have cloned a repository before should rather fetch the latest changes.
+ This is (sometimes dramatically!) faster for the client and less expensive for the server.
+
+
+ High numbers of clone operations usually indicate an unfavorably configured CI process.
+
+
+
+
+
+
Transferred Data
+
+
+
+ The amount of data transferred for clones and fetches (excluding Git LFS traffic) as well as pure Git LFS traffic (indirectly triggered through clones and fetches).
+
+
+ To save resources, you should try to minimize the number of clones, especially for large repositories.
+
+
+
+
+
+
+
+
+ This table lists the repositories that accounted for the majority of the traffic yesterday.
+
+
+
diff --git a/docs/repository/index.html b/docs/repository/index.html
new file mode 100644
index 00000000..fa8893bb
--- /dev/null
+++ b/docs/repository/index.html
@@ -0,0 +1,10 @@
+---
+layout: repository
+title: Repository
+permalink: /repository/
+---
+
+
diff --git a/updater/config.py.example b/updater/config.py.example
index c4857ad7..93118369 100755
--- a/updater/config.py.example
+++ b/updater/config.py.example
@@ -64,6 +64,7 @@ configuration = \
"github-enterprise",
],
+
# The collaboration metric calculates the membership of users to
# organizations based on the number of pushes. A user is called "a member
# of an organization" if the user has made the most pushes to that
@@ -72,6 +73,13 @@ configuration = \
"memberlessOrganizations": [],
+ # Name with owner of repositories that are to be monitored for repository
+ # specific metrics.
+ #
+ # Example: [ "test-org/large-repo", "it/mobile-client" ]
+ "monitoredRepositories": [],
+
+
# Help Links
#
# [1] https://help.github.com/enterprise/2.11/admin/guides/installation/administrative-shell-ssh-access/
@@ -95,4 +103,7 @@ configuration = \
# Command that returns the SQL shell with the correct environment
"databaseCommand": ["ghe-dbconsole", "-y"],
+ # Path to git-sizer (https://github.com/github/git-sizer) used for Git metrics.
+ # Default is home directory of admin user on GitHub Enterprise appliance.
+ "gitSizerPath": "/home/admin/git-sizer"
}
diff --git a/updater/helpers.py b/updater/helpers.py
index d8fe9ea6..bf8468f9 100644
--- a/updater/helpers.py
+++ b/updater/helpers.py
@@ -11,8 +11,13 @@ def __str__(self):
return "%0.3f" % self
# Executes a single command and returns stdout and stderr
-def executeCommand(command, stdin = None, cwd = None):
- with subprocess.Popen(command, stdout = subprocess.PIPE, stderr = subprocess.PIPE, stdin = (subprocess.PIPE if stdin != None else None), cwd = cwd) as process:
+def executeCommand(command, stdin = None, cwd = None, repository = None):
+ env = os.environ.copy()
+
+ if repository:
+ env["REPOSITORY"] = repository
+
+ with subprocess.Popen(command, stdout = subprocess.PIPE, stderr = subprocess.PIPE, stdin = (subprocess.PIPE if stdin != None else None), cwd = cwd, env = env) as process:
stdout, stderr = process.communicate(input = (stdin.encode("utf-8") if stdin != None else None))
print(stderr.decode("utf-8"), file = sys.stderr)
@@ -46,3 +51,10 @@ def prepareDataDirectory(dataDirectory, fetchChanges = True):
else:
executeCommand(["git", "fetch"], cwd = dataDirectory)
executeCommand(["git", "reset", "--hard", "origin/master"], cwd = dataDirectory)
+
+# Create a data directory for each monitored repository if not present
+def prepareRepositoryDataDirectory(dataDirectory):
+ for repository in configuration["monitoredRepositories"]:
+ repositoryDataDirectory = os.path.join(dataDirectory, "repository", repository.split('/')[0], repository.split('/')[1])
+ if not os.path.exists(repositoryDataDirectory):
+ os.makedirs(repositoryDataDirectory)
\ No newline at end of file
diff --git a/updater/reports/Report.py b/updater/reports/Report.py
index f8e3e492..bdbfb737 100644
--- a/updater/reports/Report.py
+++ b/updater/reports/Report.py
@@ -9,19 +9,28 @@
# Abstract base class for all reports that automates reading and writing reports etc.
class Report(object):
- def __init__(self, configuration, dataDirectory, metaStats):
+ def __init__(self, configuration, dataDirectory, metaStats, repository = None):
self.configuration = configuration
self.dataDirectory = dataDirectory
self.metaStats = metaStats
+ self.repository = repository
self.header = None
self.data = []
self.detailedHeader = None
self.detailedData = None
+ if self.repository != None:
+ self.repositoryOwner = self.repository.split('/')[0]
+ self.repositoryName = self.repository.split('/')[1]
+
# Name of the report, must be overridden in subclasses
def name(self):
return "unnamed-report"
+ # Name of the report within the meta stats report
+ def metaName(self):
+ return self.name()
+
# Determines the output filename of the report
def fileName(self):
return os.path.join(self.dataDirectory, self.name() + ".tsv")
@@ -69,6 +78,10 @@ def executeScript(self, script, stdin = None):
# If the script is a file, then read its content as-is into stdin
with open(script) as f:
stdin = f.read()
+
+ # Replace $REPOSITORY in script with repository (nwo) if present
+ if self.repository:
+ stdin = stdin.replace("$REPOSITORY", self.repository)
except:
# If the script is a list of strings, then escape the content and set it to stdin
stdin = " ".join(map(lambda x: '"' + x.replace('\\"', '\\\\"').replace('"', '\\"') + '"', script))
@@ -82,7 +95,7 @@ def executeScript(self, script, stdin = None):
"admin@" + self.configuration["remoteRun"]["gheHost"]
] + script
- stdout, stderr = executeCommand(script, stdin)
+ stdout, stderr = executeCommand(script, stdin = stdin, repository = self.repository)
print(stderr.decode("utf-8"), file = sys.stderr)
sys.stderr.flush()
@@ -159,7 +172,7 @@ def updateData(self):
# Performs the update by reading existing data, updating it (as defined by subclass), and writing the result
def update(self):
- print("Started update of " + self.name() + ".tsv", file = sys.stderr)
+ print("Started update of " + self.metaName() + ".tsv", file = sys.stderr)
sys.stderr.flush()
timeStart = time.time()
@@ -169,9 +182,9 @@ def update(self):
self.writeData()
timeElapsed = PrettyFloat(time.time() - timeStart)
- self.metaStats["runtimes"].append([self.name(), timeElapsed])
+ self.metaStats["runtimes"].append([self.metaName(), timeElapsed])
- print("Finished update of " + self.name() + ".tsv (runtime: " + str(timeElapsed) + " s)", file = sys.stderr)
+ print("Finished update of " + self.metaName() + ".tsv (runtime: " + str(timeElapsed) + " s)", file = sys.stderr)
sys.stderr.flush()
def andExcludedEntities(self, column, delimiter = "AND"):
diff --git a/updater/reports/ReportReposMonitored.py b/updater/reports/ReportReposMonitored.py
new file mode 100644
index 00000000..8ca83db4
--- /dev/null
+++ b/updater/reports/ReportReposMonitored.py
@@ -0,0 +1,14 @@
+from .Report import *
+
+# Lists monitored repositories
+class ReportReposMonitored(Report):
+ def name(self):
+ return "repository-monitored"
+
+ # The data is overwritten every day, so skip reading the old data
+ def readData(self):
+ pass
+
+ def updateData(self):
+ self.header = ['repository']
+ self.data = map(lambda repository: [repository], self.configuration["monitoredRepositories"])
diff --git a/updater/reports/repository/ReportGitHubApiRequestTypesByCount.py b/updater/reports/repository/ReportGitHubApiRequestTypesByCount.py
new file mode 100644
index 00000000..3dfa9e40
--- /dev/null
+++ b/updater/reports/repository/ReportGitHubApiRequestTypesByCount.py
@@ -0,0 +1,15 @@
+from .ReportRepository import *
+
+# Lists the number of GitHub api request types
+# Ordered by count in descending order
+class ReportGitHubApiRequestTypesByCount(ReportRepository):
+ def name(self):
+ return "github-api-request-types-by-count"
+
+ # The data is overwritten every day, so skip reading the old data
+ def readData(self):
+ pass
+
+ def updateData(self):
+ self.header, self.data = self.parseData(
+ self.executeScript(self.scriptPath("repository/github-api-request-types-by-count.sh")))
\ No newline at end of file
diff --git a/updater/reports/repository/ReportGitHubGitDownload.py b/updater/reports/repository/ReportGitHubGitDownload.py
new file mode 100644
index 00000000..a2404b07
--- /dev/null
+++ b/updater/reports/repository/ReportGitHubGitDownload.py
@@ -0,0 +1,17 @@
+import os
+
+from ..ReportGitDownload import *
+
+# Report how much data is downloaded from the GHE instance for a specific repository
+class ReportGitHubGitDownload(ReportGitDownload):
+ def name(self):
+ return "github-git-download"
+
+ def metaName(self):
+ return self.repository + "/" + self.name()
+
+ def fileName(self):
+ return os.path.join(self.dataDirectory, "repository", self.repositoryOwner, self.repositoryName, self.name() + ".tsv")
+
+ def detailedFileName(self):
+ return os.path.join(self.dataDirectory, "repository", self.repositoryOwner, self.repositoryName, self.name() + "-detailed.tsv")
\ No newline at end of file
diff --git a/updater/reports/repository/ReportGitSizer.py b/updater/reports/repository/ReportGitSizer.py
new file mode 100644
index 00000000..72e3a3b5
--- /dev/null
+++ b/updater/reports/repository/ReportGitSizer.py
@@ -0,0 +1,90 @@
+import json
+
+from .ReportRepository import *
+
+class ReportGitSizer(ReportRepository):
+ def name(self):
+ return "git-sizer"
+
+ def convertBytesToKiB(self, bytes):
+ return round(bytes / 1024, 1)
+
+ def convertBytesToMiB(self, bytes):
+ return round(bytes / 1024 ** 2, 3)
+
+ def updateData(self):
+ stdout = self.executeScript(["ghe-repo", self.repository, "-c", self.configuration["gitSizerPath"] + " --json --json-version=2"])
+
+ sizerData = json.loads(stdout.decode("utf-8"))
+
+ getSizerDataValue = lambda key: sizerData[key]["value"]
+
+ self.header = \
+ [
+ "date",
+
+ # Overall repository size
+ "commits",
+ "commits [MiB]",
+ "trees",
+ "trees [MiB]",
+ "tree entries",
+ "blobs",
+ "blobs [MiB]",
+ "annotated tags",
+ "refs",
+
+ # Biggest objects
+ "biggest commit [KiB]",
+ "most parents",
+ "most tree entries",
+ "biggest blob [MiB]",
+
+ # History structure
+ "largest history depth",
+ "largest tag depth",
+
+ # Biggest checkouts
+ "directories",
+ "path depth",
+ "path length",
+ "files",
+ "size of files [MiB]",
+ "symlinks",
+ "submodules"
+ ]
+
+ self.data.append(
+ [
+ str(self.yesterday()),
+
+ # Overall repository size
+ getSizerDataValue("uniqueCommitCount"),
+ self.convertBytesToMiB(getSizerDataValue("uniqueCommitSize")),
+ getSizerDataValue("uniqueTreeCount"),
+ self.convertBytesToMiB(getSizerDataValue("uniqueTreeSize")),
+ getSizerDataValue("uniqueTreeEntries"),
+ getSizerDataValue("uniqueBlobCount"),
+ self.convertBytesToMiB(getSizerDataValue("uniqueBlobSize")),
+ getSizerDataValue("uniqueTagCount"),
+ getSizerDataValue("referenceCount"),
+
+ # Biggest objects
+ self.convertBytesToKiB(getSizerDataValue("maxCommitSize")),
+ getSizerDataValue("maxCommitParentCount"),
+ getSizerDataValue("maxTreeEntries"),
+ self.convertBytesToMiB(getSizerDataValue("maxBlobSize")),
+
+ # History structure
+ getSizerDataValue("maxHistoryDepth"),
+ getSizerDataValue("maxTagDepth"),
+
+ # Biggest checkouts
+ getSizerDataValue("maxCheckoutTreeCount"),
+ getSizerDataValue("maxCheckoutPathDepth"),
+ getSizerDataValue("maxCheckoutPathLength"),
+ getSizerDataValue("maxCheckoutBlobCount"),
+ self.convertBytesToMiB(getSizerDataValue("maxCheckoutBlobSize")),
+ getSizerDataValue("maxCheckoutLinkCount"),
+ getSizerDataValue("maxCheckoutSubmoduleCount")
+ ])
\ No newline at end of file
diff --git a/updater/reports/repository/ReportOverview.py b/updater/reports/repository/ReportOverview.py
new file mode 100644
index 00000000..7b6cbef8
--- /dev/null
+++ b/updater/reports/repository/ReportOverview.py
@@ -0,0 +1,41 @@
+from .ReportRepository import *
+
+# Lists the name and the size of the repository
+class ReportOverview(ReportRepository):
+ def name(self):
+ return "overview"
+
+ # The data is overwritten every day, so skip reading the old data
+ def readData(self):
+ pass
+
+ def updateData(self):
+ self.header, self.data = self.parseData(
+ self.executeQuery(self.query()))
+
+ # Convert result into a list of keys and values
+ self.header = ['key', 'value']
+ name = self.data[0][0]
+ size = self.data[0][1]
+
+ # Show size in MB
+ size = str(round(int(size) / 1000)) + " MB"
+
+ self.data = [
+ ["Repository Name", name],
+ ["Repository Size", size]
+ ]
+
+ # Collect repository name and size
+ def query(self):
+ return '''
+ SELECT
+ CONCAT(users.login, "/", repositories.name) AS repository,
+ repositories.disk_usage
+ FROM
+ repositories
+ JOIN users on users.id = repositories.owner_id
+ WHERE
+ users.login = "''' + self.repositoryOwner + '''"
+ AND repositories.name = "''' + self.repositoryName + '''"
+ '''
\ No newline at end of file
diff --git a/updater/reports/repository/ReportRepository.py b/updater/reports/repository/ReportRepository.py
new file mode 100644
index 00000000..5d7e5aac
--- /dev/null
+++ b/updater/reports/repository/ReportRepository.py
@@ -0,0 +1,9 @@
+from ..Report import *
+
+# Abstract base class for all repository reports
+class ReportRepository(Report):
+ def metaName(self):
+ return self.repository + "/" + self.name()
+
+ def fileName(self):
+ return os.path.join(self.dataDirectory, "repository", self.repositoryOwner, self.repositoryName, self.name() + ".tsv")
\ No newline at end of file
diff --git a/updater/scripts/api-requests-by-user.sh b/updater/scripts/api-requests-by-user.sh
old mode 100644
new mode 100755
diff --git a/updater/scripts/api-requests.sh b/updater/scripts/api-requests.sh
old mode 100644
new mode 100755
diff --git a/updater/scripts/git-download.sh b/updater/scripts/git-download.sh
old mode 100644
new mode 100755
index 534bf82a..ff1ec61b
--- a/updater/scripts/git-download.sh
+++ b/updater/scripts/git-download.sh
@@ -3,6 +3,19 @@
# Calculate download traffic per day
#
+# Filter by a specific repository if $REPOSITORY is defined or replaced
+REPO_NAME="$REPOSITORY"
+
+if [ -n "$REPO_NAME" ]; then
+ # Escape forward slash
+ REPO_NAME="${REPO_NAME/\//\\/}"
+
+ REPO_NAME_FIELD="\"repo_name\":\"($REPO_NAME)\".*"
+else
+ REPO_NAME_FIELD='"repo_name":"([^"]+).*'
+fi
+
+
function ghe_greater_equal () {
cat /etc/github/enterprise-release |
perl -sne '
@@ -44,7 +57,7 @@ if ghe_greater_equal "2.12.0" ; then
'print if s/.*' \
'"cloning":([^,]+).*' \
'"program":"upload-pack".*' \
- '"repo_name":"([^"]+).*' \
+ $REPO_NAME_FIELD \
'"uploaded_bytes":([^,]+).*' \
'"user_login":"([^"]+).*' \
'/\2\t\4\t\1\t\3/'
@@ -52,7 +65,7 @@ else
printf -v EXTRACT_FIELDS "%s" \
'print if s/.*' \
'"program":"upload-pack".*' \
- '"repo_name":"([^"]+).*' \
+ $REPO_NAME_FIELD \
'"user_login":"([^"]+).*' \
'"cloning":([^,]+).*' \
'"uploaded_bytes":([^ ]+).*' \
diff --git a/updater/scripts/git-lfs-download.sh b/updater/scripts/git-lfs-download.sh
old mode 100644
new mode 100755
diff --git a/updater/scripts/git-requests.sh b/updater/scripts/git-requests.sh
old mode 100644
new mode 100755
diff --git a/updater/scripts/git-versions.sh b/updater/scripts/git-versions.sh
old mode 100644
new mode 100755
diff --git a/updater/scripts/repository/github-api-request-types-by-count.sh b/updater/scripts/repository/github-api-request-types-by-count.sh
new file mode 100755
index 00000000..c9a7b2af
--- /dev/null
+++ b/updater/scripts/repository/github-api-request-types-by-count.sh
@@ -0,0 +1,20 @@
+#!/bin/bash
+
+#
+# Count number of API calls per route
+#
+
+echo -e "type\tcount"
+
+zcat -f /var/log/github/unicorn.log.1* |
+ grep -F 'request_category=api' |
+ grep -Fv 'remote_address=127.0.0.1' |
+ grep -Fv "controller=\"Api::Internal" |
+ grep "repo=$REPOSITORY " |
+ grep -oP 'route=\K\S+' |
+ grep -Fvx 'nil' |
+ sort |
+ uniq -ic |
+ sort -rn |
+ head -20 |
+ awk '{ printf("%s\t%s\n", $2, $1) }'
diff --git a/updater/scripts/tokenless-auth.sh b/updater/scripts/tokenless-auth.sh
old mode 100644
new mode 100755
diff --git a/updater/update-stats.py b/updater/update-stats.py
index 5264a322..221c2913 100755
--- a/updater/update-stats.py
+++ b/updater/update-stats.py
@@ -28,12 +28,17 @@
from reports.ReportPRUsage import *
from reports.ReportRepoActivity import *
from reports.ReportRepositoryHistory import *
+from reports.ReportReposMonitored import *
from reports.ReportReposPersonalNonOwnerPushes import *
from reports.ReportRepoUsage import *
from reports.ReportTeamsLegacy import *
from reports.ReportTeamsTotal import *
from reports.ReportTokenlessAuth import *
from reports.ReportUsers import *
+from reports.repository.ReportOverview import *
+from reports.repository.ReportGitSizer import *
+from reports.repository.ReportGitHubGitDownload import *
+from reports.repository.ReportGitHubApiRequestTypesByCount import *
def writeMeta(dataDirectory):
outputFilePath = os.path.join(dataDirectory, "meta.tsv")
@@ -67,6 +72,7 @@ def main():
# Prepare the data directory for writing the new data
dataDirectory = locateDataDirectory()
prepareDataDirectory(dataDirectory, fetchChanges = not configuration["dryRun"])
+ prepareRepositoryDataDirectory(dataDirectory)
# Verify schema version for forward compatibility
checkSchemaVersion(dataDirectory)
@@ -94,6 +100,7 @@ def main():
ReportPRUsage(configuration, dataDirectory, metaStats).update()
ReportRepoActivity(configuration, dataDirectory, metaStats).update()
ReportRepositoryHistory(configuration, dataDirectory, metaStats).update()
+ ReportReposMonitored(configuration, dataDirectory, metaStats).update()
ReportReposPersonalNonOwnerPushes(configuration, dataDirectory, metaStats).update()
ReportRepoUsage(configuration, dataDirectory, metaStats).update()
ReportTeamsLegacy(configuration, dataDirectory, metaStats).update()
@@ -101,6 +108,13 @@ def main():
ReportTokenlessAuth(configuration, dataDirectory, metaStats).update()
ReportUsers(configuration, dataDirectory, metaStats).update()
+ # Repository reports
+ for repository in configuration["monitoredRepositories"]:
+ ReportOverview(configuration, dataDirectory, metaStats, repository).update()
+ ReportGitSizer(configuration, dataDirectory, metaStats, repository).update()
+ ReportGitHubGitDownload(configuration, dataDirectory, metaStats, repository).update()
+ ReportGitHubApiRequestTypesByCount(configuration, dataDirectory, metaStats, repository).update()
+
# Write meta infos
writeMeta(dataDirectory)
writeMetaStats(metaStats, dataDirectory)