Commit 709d882d authored by Hugo Posnic's avatar Hugo Posnic

Rewrite

parent 315a31ef
*~
*.pro.user
CMakeLists.txt.user
build
build-debug
# Byte-compiled / optimized / DLL files
__pycache__/
*.py[cod]
*$py.class
# C extensions
*.so
# Distribution / packaging
.Python
env/
build/
develop-eggs/
dist/
downloads/
eggs/
.eggs/
lib/
lib64/
parts/
sdist/
var/
wheels/
*.egg-info/
.installed.cfg
*.egg
# PyInstaller
# Usually these files are written by a python script from a template
# before PyInstaller builds the exe, so as to inject date/other infos into it.
*.manifest
*.spec
# Installer logs
pip-log.txt
pip-delete-this-directory.txt
# Unit test / coverage reports
htmlcov/
.tox/
.coverage
.coverage.*
.cache
nosetests.xml
coverage.xml
*,cover
.hypothesis/
# Translations
*.mo
*.pot
# Django stuff:
*.log
local_settings.py
# Flask stuff:
instance/
.webassets-cache
# Scrapy stuff:
.scrapy
# Sphinx documentation
docs/_build/
# PyBuilder
target/
# Jupyter Notebook
.ipynb_checkpoints
# pyenv
.python-version
# celery beat schedule file
celerybeat-schedule
# dotenv
.env
# virtualenv
.venv/
venv/
ENV/
# Spyder project settings
.spyderproject
# Rope project settings
.ropeproject
Main developers:
Roland Singer <roland@manjaro.org>
- Hugo Posnic <huluti@manjaro.org>
cmake_minimum_required(VERSION 2.8)
project(manjaro-web-repo)
SET (CMAKE_INSTALL_PREFIX "/usr/")
SET (WT_CONNECTOR "wthttp" CACHE STRING "Connector used (wthttp or wtfcgi)")
add_definitions(-std=c++11 -Wall)
add_subdirectory(src)
add_subdirectory(data)
INSTALL(DIRECTORY "style"
DESTINATION "share/${PROJECT_NAME}"
)
</tbody>
</table>
</div>
<script src="https://ajax.googleapis.com/ajax/libs/jquery/3.1.1/jquery.min.js" integrity="sha384-3ceskX3iaEnIogmQchP8opvBy3Mi7Ce34nWjpBIwVTHfGYWQS9jwHDVRnpKKHJg7" crossorigin="anonymous"></script>
<script src="https://cdnjs.cloudflare.com/ajax/libs/tether/1.3.7/js/tether.min.js" integrity="sha384-XTs3FgkjiBgo8qjEjBk0tGmf3wPrWtA6coPfQDfFEY8AnYJwjalXCiosYRBIBZX8" crossorigin="anonymous"></script>
<script src="https://maxcdn.bootstrapcdn.com/bootstrap/4.0.0-alpha.5/js/bootstrap.min.js" integrity="sha384-BLiI7JTZm+JWlgKa0M0kGRpJbF2J8q+qreVrKBC47e3K6BW78kGLrCkeRX6I9RoK" crossorigin="anonymous"></script>
</body>
</html>
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="utf-8">
<title>Manjaro Repository</title>
<meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no">
<meta http-equiv="x-ua-compatible" content="ie=edge">
<link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/4.0.0-alpha.5/css/bootstrap.min.css" integrity="sha384-AysaV+vQoT3kOAXZkl02PThvDr8HYKPZhNT5h/CXfBThSRXQ6jW5DO2ekP5ViFdi" crossorigin="anonymous">
<link href="data/style.css" rel="stylesheet">
<link href="https://maxcdn.bootstrapcdn.com/font-awesome/4.7.0/css/font-awesome.min.css" rel="stylesheet" integrity="sha384-wvfXpqpZZVQGK6TAh5PVlGOfQNHSoD2xbE+QkPxCAFlNEevoEH3Sl0sibVcOQVnN" crossorigin="anonymous">
<link rel="icon" href="data/manjaro.png">
</head>
<body>
<nav class="navbar navbar-fixed-top navbar-light bg-faded">
<a class="navbar-brand" href="#">Manjaro Repository</a>
<ul class="nav navbar-nav">
<li class="nav-item active">
<a class="nav-link" href="#">Repository State <span class="sr-only">(current)</span></a>
</li>
</ul>
</nav>
<div class="container">
<div class="template">
<h1><img src="data/manjaro.png" alt=""> Manjaro Repository</h1>
<p class="lead">Check state of ours mirrors.</p>
<a href="" class="btn btn-secondary"><i class="fa fa-refresh" aria-hidden="true"></i> Refresh</a>
</div>
<table class="table table-striped table-sm">
<thead>
<tr>
<th>Mirror <i class="fa fa-sort" aria-hidden="true"></i></th>
<th>Country <i class="fa fa-sort" aria-hidden="true"></i></th>
<th>Protocol <i class="fa fa-sort" aria-hidden="true"></i></th>
<th>Last sync (hh:mm) <i class="fa fa-sort" aria-hidden="true"></i></th>
<th>Stable <i class="fa fa-sort" aria-hidden="true"></i></th>
<th>Testing <i class="fa fa-sort" aria-hidden="true"></i></th>
<th>Unstable <i class="fa fa-sort" aria-hidden="true"></i></th>
</tr>
</thead>
<tbody id="mirrors">
body {
padding-top: 5rem;
}
thead, th, td {
border: 0 !important;
font-family: inherit;
}
.template {
padding: 1rem 0;
text-align: center;
}
.tableView_orange {
background-color: #FFC34A !important;
}
.tableView_yellow {
background-color: #FFFF50 !important;
}
.tableView_red {
background-color: #FF492F !important;
}
.content {
padding: 10px 10px 10px 10px;
}
.Wt-tableview {
margin: 0 auto;
}
{
"France": [
"http://manjarolinux.polymorf.frs/$branch/$repo/$arch",
"http://mirror.lignux.com/manjaro/$branch/$repo/$arch",
"http://ftp.free.org/mirrors/repo.manjaro.org/repos/$branch/$repo/$arch",
"http://fr.mirror.babylon.network/manjaro/$branch/$repo/$arch"
],
"Denmark": [
"http://mirrors.dotsrc.org/manjaro/$branch/$repo/$arch",
"http://www.uex.dk/repos/manjaro/$branch/$repo/$arch"
]
}
# Boost
find_package(Boost COMPONENTS system filesystem thread signals random regex program_options date_time REQUIRED)
# Wt
SET(WT_LIBRARIES wt)
SET(WT_INCLUDE_DIR /usr/include/Wt)
# Curl
SET(CURL_LIBRARIES curl)
SET(CURL_INCLUDE_DIR /usr/include/curl)
# Project
include_directories(${Boost_INCLUDE_DIR} ${WT_INCLUDE_DIR} ${CURL_INCLUDE_DIR})
link_directories(${Boost_LIBRARY_DIR})
aux_source_directory(. SRC_FILES)
add_executable(${PROJECT_NAME} ${SRC_FILES})
target_link_libraries(${PROJECT_NAME} ${Boost_LIBRARIES} ${WT_CONNECTOR} ${WT_LIBRARIES} ${CURL_LIBRARIES})
set_target_properties(${PROJECT_NAME} PROPERTIES OUTPUT_NAME ${PROJECT_NAME}.wt)
INSTALL(TARGETS ${PROJECT_NAME}
RUNTIME DESTINATION bin
)
/*
* manjaro-web-repo
* Roland Singer <roland@manjaro.org>
*
* Copyright (C) 2007 Free Software Foundation, Inc.
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "backgroundworker.h"
BackgroundWorker::BackgroundWorker()
{
}
BackgroundWorker::~BackgroundWorker()
{
}
void BackgroundWorker::start()
{
m_Thread = boost::thread(&BackgroundWorker::run, this);
}
void BackgroundWorker::run()
{
boost::posix_time::minutes sleepTime(15);
while (true) {
updateRepositoryStateList();
// Sleep
boost::this_thread::sleep(sleepTime);
}
}
void BackgroundWorker::updateRepositoryStateList()
{
vector<Global::RepoState> repoStates;
map<string, string> currentBranchHashStates;
WDateTime currentDateTime = WDateTime::currentDateTime();
/* Get branches */
vector<string> branches;
boost::split(branches, REPO_BRANCHES, boost::is_any_of(" "), boost::token_compress_on);
/* Get current branch hash state */
for (unsigned int i = 0; i < branches.size(); i++) {
const string branch = branches.at(i);
const string filePath = string(REPO_PATH) + "/" + branch + "/state";
ifstream file(filePath.c_str(), std::ios::in);
if (!file.is_open()) {
cerr << "error: failed to open file '" << filePath << "'!" << endl;
return;
}
string line;
while (!file.eof()) {
getline(file, line);
size_t pos = line.find_first_of('#');
if (pos != string::npos)
line.erase(pos);
boost::trim(line);
if (!boost::starts_with(line, "state") || string::npos == line.find("="))
continue;
line = line.substr(line.find('=') + 1, string::npos);
boost::trim(line);
currentBranchHashStates[branch] = line;
}
file.close();
}
/* Get git source */
string cmd, workingDir;
string tmpPath = string(TMP_PATH) + "/git";;
if (boost::filesystem::exists(tmpPath)) {
workingDir = tmpPath + "/" + string(GIT_NAME);
cmd = "git pull origin master";
}
else {
if (!boost::filesystem::create_directories(tmpPath)) {
cerr << "error: failed to create git tmp path!" << endl;
return;
}
workingDir = tmpPath;
cmd = "git clone " + string(GIT_URL);
}
if (!Global::executeCommand(cmd, workingDir)) {
cerr << "error: git process failed" << endl;
return;
}
/* Get mirrors and time */
boost::filesystem::directory_iterator end_itr;
for(boost::filesystem::directory_iterator i(tmpPath + "/" + MIRRORS_PATH); i != end_itr; ++i)
{
// Skip if not a file
if(!boost::filesystem::is_regular_file(i->status()))
continue;
ifstream file(i->path().string().c_str(), std::ios::in);
if (!file.is_open()) {
cerr << "error: failed to open file '" << i->path().string() << "'!" << endl;
return;
}
string country = "unknown";
string line;
while (!file.eof()) {
getline(file, line);
size_t pos = line.find_first_of('#');
if (pos != string::npos)
line.erase(pos);
boost::trim(line);
if (boost::starts_with(line, "[") && boost::ends_with(line, "]")) {
country = line.substr(1, line.length() - 2);
}
else if (boost::starts_with(line, "Server") && string::npos != line.find("=")) {
line = line.substr(line.find('=') + 1, string::npos);
boost::trim(line);
Global::RepoState repo;
repo.country = country;
repo.url = line;
for (unsigned int i = 0; i < branches.size(); i++) {
const string branch = branches.at(i);
if (currentBranchHashStates.find(branch) == currentBranchHashStates.end() ) {
repo.states[branch] = Global::STATE_UNKOWN;
continue;
}
string url = repo.url;
boost::replace_last(url, "$branch/$repo/$arch", branch + "/state");
repo.states[branch] = getRepoBranchState(url, currentBranchHashStates.at(branch));
}
repo.url = repo.url.substr(0, repo.url.find("/$branch"));
repo.lastSync = getRepoLastSync(repo.url + "/state", currentDateTime);
if (boost::starts_with(repo.url, "http://"))
repo.protocol = "http";
else if (boost::starts_with(repo.url, "ftp://"))
repo.protocol = "ftp";
else
repo.protocol = "unknown";
repoStates.push_back(repo);
}
}
file.close();
}
Global::setRepoStates(repoStates);
}
long BackgroundWorker::getRepoLastSync(const string & url, const WDateTime & currentDateTime)
{
vector<string> split;
string webContent= Global::getWebContent(url);
boost::trim(webContent);
boost::split(split, webContent, boost::is_any_of("\n"), boost::token_compress_on);
string repoDateTimeStr;
for (unsigned int i = 0; i < split.size(); i++) {
string line = split.at(i);
line = line.substr(0, line.find('#'));
boost::trim(line);
if (!boost::starts_with(line, "date") || string::npos == line.find("="))
continue;
repoDateTimeStr = line.substr(line.find('=') + 1, string::npos);
boost::trim(repoDateTimeStr);
}
if (repoDateTimeStr.empty())
return -1;
WDateTime repoDateTime = WDateTime::fromString(repoDateTimeStr, "yyyy-MM-dd'T'hh:mm:ss");
if (repoDateTime > currentDateTime)
return -1;
return repoDateTime.secsTo(currentDateTime) / 60;
}
Global::STATE BackgroundWorker::getRepoBranchState(const string & url, const string & currentBranchHashState)
{
vector<string> split;
string webContent= Global::getWebContent(url);
boost::trim(webContent);
boost::split(split, webContent, boost::is_any_of("\n"), boost::token_compress_on);
string hashState;
for (unsigned int i = 0; i < split.size(); i++) {
string line = split.at(i);
line = line.substr(0, line.find('#'));
boost::trim(line);
if (!boost::starts_with(line, "state") || string::npos == line.find("="))
continue;
hashState = line.substr(line.find('=') + 1, string::npos);
boost::trim(hashState);
}
if (hashState.empty())
return Global::STATE_UNKOWN;
else if (hashState == currentBranchHashState)
return Global::STATE_UP_TO_DATE;
else
return Global::STATE_OUT_OF_DATE;
}
/*
* manjaro-web-repo
* Roland Singer <roland@manjaro.org>
*
* Copyright (C) 2007 Free Software Foundation, Inc.
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef BACKGROUNDWORKER_H
#define BACKGROUNDWORKER_H
#include <iostream>
#include <string>
#include <vector>
#include <map>
#include <fstream>
#include <boost/thread.hpp>
#include <boost/algorithm/string.hpp>
#include <boost/filesystem.hpp>
#include <WDateTime>
#include "const.h"
#include "global.h"
using namespace std;
using namespace Wt;
class BackgroundWorker
{
public:
BackgroundWorker();
~BackgroundWorker();
void start();
private:
boost::thread m_Thread;
void run();
void updateRepositoryStateList();
long getRepoLastSync(const string & url, const WDateTime & currentDateTime);
Global::STATE getRepoBranchState(const string & url, const string & currentBranchHashState);
};
#endif // BACKGROUNDWORKER_H
#!/usr/bin/env python3
import json
class Builder():
def __init__(self, states):
self.states = states
def write_json_output(self):
try:
with open("status.json", "w") as json_output:
json.dump(self.states, json_output)
print("JSON output save in status.json")
except OSError:
print("Can't write JSON output")
def write_html_output(self):
try:
with open("data/header.html", "r") as header_file:
header = header_file.read()
except OSError:
print("Can't read HTML header file")
try:
with open("data/footer.html", "r") as footer_file:
footer = footer_file.read()
except OSError:
print("Can't read HTML footer file")
try:
with open("index.html", "w") as index_file:
content = header
for state in self.states:
if state["branches"]["stable"] and state["branches"]["testing"] and state["branches"]["unstable"]:
color = "table-success"
elif state["branches"]["stable"] or state["branches"]["testing"] or state["branches"]["unstable"]:
color = "table-warning"
else:
color = "table-danger"
content += "<tr class=\"" + color + "\">"
content += "<tr>"
content += "<td><a href=\"" + state["url"] + "\">" + state["url"] + "</a></td>"
content += "<td>" + state['country'] + "</td>"
content += "<td>" + state["protocol"] + "</td>"
content += "<td>" + state["last_sync"] + "</td>"
for branch in state["branches"]:
if state["branches"][branch]:
content +="<td><i class=\"fa fa-check\" aria-hidden=\"true\"></i></td>";
else:
content += "<td><i class=\"fa fa-times\" aria-hidden=\"true\"></i></td>";
content += "</tr>"
content += footer
index_file.write(content)
print("HTML output save in index.html")
except OSError:
print("Can't write HTML output")
MIRRORS_URL = "mirrors.json"
BRANCHES = ("stable", "testing", "unstable")
PROTOCOLS = ("ftp, https", "http")
SLEEP_TIME = 15
/*
* manjaro-web-repo
* Roland Singer <roland@manjaro.org>
*
* Copyright (C) 2007 Free Software Foundation, Inc.
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of