Commit 9cb8ac42 authored by guinux's avatar guinux

improved parallel downloads

parent 2a83ed03
......@@ -11,13 +11,13 @@ RefreshPeriod = 6
#NoUpdateHideIcon
## Allow Pamac to search and install packages from AUR:
EnableAUR
#EnableAUR
## When AUR support is enabled check for updates from AUR:
CheckAURUpdates
#CheckAURUpdates
## When check updates from AUR support is enabled check for vcs updates:
CheckAURVCSUpdates
#CheckAURVCSUpdates
## AUR build directory:
BuildDirectory = /var/tmp
......@@ -30,3 +30,6 @@ KeepNumPackages = 3
## Download updates in background
#DownloadUpdates
## Maximum Parallel Downloads
MaxParallelDownloads = 4
......@@ -7,7 +7,7 @@ msgid ""
msgstr ""
"Project-Id-Version: Pamac\n"
"Report-Msgid-Bugs-To: guillaume@manjaro.org\n"
"POT-Creation-Date: 2018-10-27 11:19+0200\n"
"POT-Creation-Date: 2018-11-02 12:33+0100\n"
"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
"Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
"Language-Team: LANGUAGE <LL@li.org>\n"
......@@ -157,6 +157,11 @@ msgstr ""
msgid "Checking %s dependencies"
msgstr ""
#: src/transaction.vala
#, c-format
msgid "key %s could not be imported"
msgstr ""
#: src/transaction.vala
msgid "Preparing"
msgstr ""
......@@ -245,10 +250,6 @@ msgstr ""
msgid "Configuring %s"
msgstr ""
#: src/transaction.vala
msgid "Downloading"
msgstr ""
#: src/transaction.vala
#, c-format
msgid "Downloading %s"
......@@ -341,6 +342,30 @@ msgstr ""
msgid "Choose a provider for %s"
msgstr ""
#: src/transaction-gtk.vala
msgid "Import PGP key"
msgstr ""
#: src/transaction-gtk.vala
msgid "Trust and Import"
msgstr ""
#: src/transaction-gtk.vala src/manager_window.vala
#: resources/transaction_sum_dialog.ui resources/manager_window.ui
#: resources/choose_pkgs_dialog.ui
msgid "_Cancel"
msgstr ""
#: src/transaction-gtk.vala src/transaction-cli.vala
#, c-format
msgid "The PGP key %s is needed to verify %s source files"
msgstr ""
#: src/transaction-gtk.vala src/transaction-cli.vala
#, c-format
msgid "Trust %s and import the PGP key"
msgstr ""
#: src/transaction-gtk.vala src/transaction-cli.vala
msgid "To remove"
msgstr ""
......@@ -378,12 +403,6 @@ msgstr ""
msgid "Save"
msgstr ""
#: src/transaction-gtk.vala src/manager_window.vala
#: resources/transaction_sum_dialog.ui resources/manager_window.ui
#: resources/choose_pkgs_dialog.ui
msgid "_Cancel"
msgstr ""
#: src/transaction-gtk.vala resources/progress_dialog.ui
#: resources/history_dialog.ui resources/preferences_dialog.ui
msgid "_Close"
......@@ -672,6 +691,10 @@ msgstr ""
msgid "How often to check for updates, value in hours"
msgstr ""
#: src/preferences_dialog.vala resources/preferences_dialog.ui
msgid "Maximun parallel downloads"
msgstr ""
#: src/preferences_dialog.vala resources/preferences_dialog.ui
msgid "Number of versions of each package to keep in the cache"
msgstr ""
......
......@@ -8,9 +8,15 @@
<property name="step_increment">1</property>
<property name="page_increment">10</property>
</object>
<object class="GtkAdjustment" id="max_parallel_downloads_adjustment">
<property name="upper">10</property>
<property name="value">4</property>
<property name="step_increment">2</property>
<property name="page_increment">10</property>
</object>
<object class="GtkAdjustment" id="refresh_period_adjustment">
<property name="lower">1</property>
<property name="upper">1000</property>
<property name="upper">168</property>
<property name="value">6</property>
<property name="step_increment">1</property>
<property name="page_increment">10</property>
......@@ -154,6 +160,52 @@
<property name="position">1</property>
</packing>
</child>
<child>
<object class="GtkBox" id="box4">
<property name="visible">True</property>
<property name="can_focus">False</property>
<property name="spacing">12</property>
<child>
<object class="GtkLabel" id="max_parallel_downloads_label">
<property name="visible">True</property>
<property name="can_focus">False</property>
<property name="halign">start</property>
<property name="hexpand">True</property>
<property name="wrap">True</property>
<property name="max_width_chars">300</property>
</object>
<packing>
<property name="expand">False</property>
<property name="fill">True</property>
<property name="position">0</property>
</packing>
</child>
<child>
<object class="GtkSpinButton" id="max_parallel_downloads_spin_button">
<property name="visible">True</property>
<property name="can_focus">True</property>
<property name="tooltip_text" translatable="yes">Maximun parallel downloads</property>
<property name="halign">end</property>
<property name="caps_lock_warning">False</property>
<property name="input_purpose">number</property>
<property name="adjustment">max_parallel_downloads_adjustment</property>
<property name="numeric">True</property>
<property name="update_policy">if-valid</property>
<property name="value">4</property>
</object>
<packing>
<property name="expand">False</property>
<property name="fill">True</property>
<property name="position">1</property>
</packing>
</child>
</object>
<packing>
<property name="expand">False</property>
<property name="fill">True</property>
<property name="position">2</property>
</packing>
</child>
<child>
<object class="GtkBox" id="box8">
<property name="visible">True</property>
......@@ -192,7 +244,7 @@
<packing>
<property name="expand">False</property>
<property name="fill">True</property>
<property name="position">2</property>
<property name="position">3</property>
</packing>
</child>
<child>
......@@ -240,7 +292,7 @@
<packing>
<property name="expand">False</property>
<property name="fill">True</property>
<property name="position">3</property>
<property name="position">4</property>
</packing>
</child>
<child>
......@@ -258,7 +310,7 @@
<packing>
<property name="expand">False</property>
<property name="fill">True</property>
<property name="position">4</property>
<property name="position">5</property>
</packing>
</child>
<child>
......@@ -276,7 +328,7 @@
<packing>
<property name="expand">False</property>
<property name="fill">True</property>
<property name="position">5</property>
<property name="position">6</property>
</packing>
</child>
<child>
......@@ -399,7 +451,7 @@
<packing>
<property name="expand">True</property>
<property name="fill">True</property>
<property name="position">6</property>
<property name="position">7</property>
</packing>
</child>
</object>
......
......@@ -971,7 +971,7 @@ namespace Pamac {
return true;
}
void download_files () {
void download_files (uint64 max_parallel_downloads) {
// use to track downloads progress
uint timeout_id = Timeout.add (500, compute_multi_download_progress);
multi_progress = new HashTable<string, uint64?> (str_hash, str_equal);
......@@ -1023,20 +1023,22 @@ namespace Pamac {
emit_event (Alpm.Event.Type.RETRIEVE_START, 0, {});
emit_multi_download (0, total_download);
// create a thread pool which will download files
// there will be one thread per mirror
// there will be two threads per mirror
try {
var dload_thread_pool = new ThreadPool<DownloadServer>.with_owned_data (
// call alpm_action.run () on thread start
(download_server) => {
download_server.download_files ();
},
// 5 simultaneous downloads
5,
// max simultaneous threads = max simultaneous downloads
(int) max_parallel_downloads,
// exclusive threads
true
);
mirrors_table.foreach_steal ((mirror, repo_set) => {
try {
// two connections per mirror
dload_thread_pool.add (new DownloadServer (mirror, repo_set));
dload_thread_pool.add (new DownloadServer (mirror, repo_set));
} catch (ThreadError e) {
stderr.printf ("Thread Error %s\n", e.message);
......@@ -1055,9 +1057,6 @@ namespace Pamac {
}
internal void trans_commit () {
// custom parallel download
download_files ();
// real commit
add_overwrite_files ();
bool success = false;
if (to_syncfirst.length > 0) {
......@@ -1150,6 +1149,17 @@ namespace Pamac {
bool trans_commit_real () {
current_error = ErrorInfos ();
bool success = true;
var config = new Config ("/etc/pamac.conf");
if (config.max_parallel_downloads >= 2) {
// custom parallel downloads
download_files (config.max_parallel_downloads);
if (cancellable.is_cancelled ()) {
trans_release ();
trans_commit_finished (false);
return false;
}
}
// real commit
Alpm.List err_data;
if (alpm_handle.trans_commit (out err_data) == -1) {
Alpm.Errno errno = alpm_handle.errno ();
......
......@@ -30,6 +30,7 @@ namespace Pamac {
public bool check_aur_updates { get; set; }
public bool check_aur_vcs_updates { get; set; }
public bool download_updates { get; set; }
public uint64 max_parallel_downloads { get; set; }
public uint64 clean_keep_num_pkgs { get; set; }
public bool clean_rm_only_uninstalled { get; set; }
public unowned HashTable<string,string> environment_variables {
......@@ -75,12 +76,14 @@ namespace Pamac {
public void reload () {
// set default options
recurse = false;
refresh_period = 6;
no_update_hide_icon = false;
enable_aur = false;
aur_build_dir = "/var/tmp";
check_aur_updates = false;
check_aur_vcs_updates = false;
download_updates = false;
max_parallel_downloads = 1;
clean_keep_num_pkgs = 3;
clean_rm_only_uninstalled = false;
parse_file (conf_path);
......@@ -95,6 +98,14 @@ namespace Pamac {
} else if (check_aur_updates == false) {
check_aur_vcs_updates = false;
}
// limited max_parallel_downloads
if (max_parallel_downloads > 10) {
max_parallel_downloads = 10;
}
// check updates at least once a week
if (refresh_period > 168) {
refresh_period = 168;
}
}
void parse_file (string path) {
......@@ -146,6 +157,11 @@ namespace Pamac {
check_aur_vcs_updates = true;
} else if (key == "DownloadUpdates") {
download_updates = true;
} else if (key == "MaxParallelDownloads") {
if (splitted.length == 2) {
unowned string val = splitted[1]._strip ();
max_parallel_downloads = uint64.parse (val);
}
}
}
} catch (GLib.Error e) {
......@@ -270,6 +286,13 @@ namespace Pamac {
} else {
data.append (line + "\n");
}
} else if (line.contains ("MaxParallelDownloads")) {
if (new_conf.lookup_extended ("MaxParallelDownloads", null, out variant)) {
data.append ("MaxParallelDownloads = %llu\n".printf (variant.get_uint64 ()));
new_conf.remove ("MaxParallelDownloads");
} else {
data.append (line + "\n");
}
} else {
data.append (line + "\n");
}
......@@ -337,6 +360,8 @@ namespace Pamac {
} else {
data.append ("#DownloadUpdates\n");
}
} else if (key == "MaxParallelDownloads") {
data.append ("MaxParallelDownloads = %llu\n".printf (val.get_uint64 ()));
}
}
}
......
......@@ -33,6 +33,10 @@ namespace Pamac {
[GtkChild]
Gtk.SpinButton refresh_period_spin_button;
[GtkChild]
Gtk.Label max_parallel_downloads_label;
[GtkChild]
Gtk.SpinButton max_parallel_downloads_spin_button;
[GtkChild]
Gtk.CheckButton no_update_hide_icon_checkbutton;
[GtkChild]
Gtk.CheckButton download_updates_checkbutton;
......@@ -75,6 +79,7 @@ namespace Pamac {
this.transaction = transaction;
refresh_period_label.set_markup (dgettext (null, "How often to check for updates, value in hours") +":");
max_parallel_downloads_label.set_markup (dgettext (null, "Maximun parallel downloads") +":");
cache_keep_nb_label.set_markup (dgettext (null, "Number of versions of each package to keep in the cache") +":");
aur_build_dir_label.set_markup (dgettext (null, "Build directory") +":");
remove_unrequired_deps_button.active = transaction.database.config.recurse;
......@@ -94,6 +99,7 @@ namespace Pamac {
refresh_period_spin_button.value = transaction.database.config.refresh_period;
previous_refresh_period = transaction.database.config.refresh_period;
}
max_parallel_downloads_spin_button.value = transaction.database.config.max_parallel_downloads;
no_update_hide_icon_checkbutton.active = transaction.database.config.no_update_hide_icon;
download_updates_checkbutton.active = transaction.database.config.download_updates;
cache_keep_nb_spin_button.value = transaction.database.config.clean_keep_num_pkgs;
......@@ -110,6 +116,7 @@ namespace Pamac {
transaction.write_alpm_config_finished.connect (on_write_alpm_config_finished);
check_updates_button.state_set.connect (on_check_updates_button_state_set);
refresh_period_spin_button.value_changed.connect (on_refresh_period_spin_button_value_changed);
max_parallel_downloads_spin_button.value_changed.connect (on_max_parallel_downloads_spin_button_value_changed);
no_update_hide_icon_checkbutton.toggled.connect (on_no_update_hide_icon_checkbutton_toggled);
download_updates_checkbutton.toggled.connect (on_download_updates_checkbutton_toggled);
cache_keep_nb_spin_button.value_changed.connect (on_cache_keep_nb_spin_button_value_changed);
......@@ -184,6 +191,12 @@ namespace Pamac {
transaction.start_write_pamac_config (new_pamac_conf);
}
void on_max_parallel_downloads_spin_button_value_changed () {
var new_pamac_conf = new HashTable<string,Variant> (str_hash, str_equal);
new_pamac_conf.insert ("MaxParallelDownloads", new Variant.uint64 (max_parallel_downloads_spin_button.get_value_as_int ()));
transaction.start_write_pamac_config (new_pamac_conf);
}
void on_cache_keep_nb_spin_button_value_changed () {
var new_pamac_conf = new HashTable<string,Variant> (str_hash, str_equal);
new_pamac_conf.insert ("KeepNumPackages", new Variant.uint64 (cache_keep_nb_spin_button.get_value_as_int ()));
......
......@@ -942,7 +942,6 @@ namespace Pamac {
break;
case 25: //Alpm.Event.Type.RETRIEVE_START
start_downloading ();
emit_action (dgettext (null, "Downloading") + "...");
break;
case 26: //Alpm.Event.Type.RETRIEVE_DONE
case 27: //Alpm.Event.Type.RETRIEVE_FAILED
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment