database.vala

/*
 *  pamac-vala
 *
 *  Copyright (C) 2019 Guillaume Benoit <guillaume@manjaro.org>
 *
 *  This program is free software; you can redistribute it and/or modify
 *  it under the terms of the GNU General Public License as published by
 *  the Free Software Foundation; either version 3 of the License, or
 *  (at your option) any later version.
 *
 *  This program is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *  GNU General Public License for more details.
 *
 *  You should have received a get of the GNU General Public License
 *  along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */

namespace Pamac {
	public class Database: Object {
		AlpmConfig alpm_config;
		Alpm.Handle? alpm_handle;
		Alpm.Handle? files_handle;
		HashTable<string, Json.Array> aur_search_results;
		HashTable<string, Json.Object> aur_infos;
		As.Store app_store;
		string locale;

		class AURUpdates {
			public List<AURPackage> updates;
			public List<AURPackage> outofdate;
			public AURUpdates (owned List<AURPackage> updates, owned List<AURPackage> outofdate) {
				this.updates = (owned) updates;
				this.outofdate = (owned) outofdate;
			}
		}

		public signal void get_updates_progress (uint percent);
		public signal void refreshed ();

		public Config config { get; construct set; }

		public Database (Config config) {
			Object (config: config);
		}

		construct {
			refresh ();
			aur_search_results = new HashTable<string, Json.Array> (str_hash, str_equal);
			aur_infos = new HashTable<string, Json.Object> (str_hash, str_equal);
			// init appstream
			app_store = new As.Store ();
			app_store.set_add_flags (As.StoreAddFlags.USE_UNIQUE_ID
									| As.StoreAddFlags.ONLY_NATIVE_LANGS
									| As.StoreAddFlags.USE_MERGE_HEURISTIC);
			locale = Environ.get_variable (Environ.get (), "LANG");
			if (locale != null) {
				// remove .UTF-8 from locale
				locale = locale.split (".")[0];
			} else {
				locale = "C";
			}
			// load alpm databases in memory
			unowned Alpm.List<unowned Alpm.Package> pkgcache = alpm_handle.localdb.pkgcache;
			unowned Alpm.List<unowned Alpm.Group> groupcache = alpm_handle.localdb.groupcache;
			unowned Alpm.List<unowned Alpm.DB> syncdbs = alpm_handle.syncdbs;
			while (syncdbs != null) {
				unowned Alpm.DB db = syncdbs.data;
				pkgcache = db.pkgcache;
				groupcache = db.groupcache;
				syncdbs.next ();
			}
		}

		public void enable_appstream () {
			try {
				app_store.load (As.StoreLoadFlags.APP_INFO_SYSTEM);
				app_store.set_search_match (As.AppSearchMatch.PKGNAME
											| As.AppSearchMatch.DESCRIPTION
											| As.AppSearchMatch.COMMENT
											| As.AppSearchMatch.NAME
											| As.AppSearchMatch.KEYWORD);
			} catch (Error e) {
				stderr.printf ("Error: %s\n", e.message);
			}
		}

		public void refresh () {
			alpm_config = new AlpmConfig ("/etc/pacman.conf");
			alpm_handle = alpm_config.get_handle ();
			if (alpm_handle == null) {
				critical (dgettext (null, "Failed to initialize alpm library"));
				return;
			} else {
				files_handle = alpm_config.get_handle (true);
			}
			refreshed ();
		}

		public List<string> get_mirrors_countries () {
			var countries = new List<string> ();
			try {
				string countries_str;
				int status;
				Process.spawn_command_line_sync ("pacman-mirrors -l",
											out countries_str,
											null,
											out status);
				if (status == 0) {
					foreach (unowned string country in countries_str.split ("\n")) {
						if (country != "") {
							countries.append (country);
						}
					}
				}
			} catch (SpawnError e) {
				stderr.printf ("Error: %s\n", e.message);
			}
			return countries;
		}

		public string get_mirrors_choosen_country () {
			string country = "";
			try {
				string countries_str;
				int status;
				Process.spawn_command_line_sync ("pacman-mirrors -lc",
											out countries_str,
											null,
											out status);
				if (status == 0) {
					// only take first country
					country = countries_str.split ("\n", 2)[0];
				}
			} catch (SpawnError e) {
				stderr.printf ("Error: %s\n", e.message);
			}
			return country;
		}

		public string get_alpm_dep_name (string dep_string) {
			return Alpm.Depend.from_string (dep_string).name;
		}

		public bool get_checkspace () {
			return alpm_handle.checkspace == 1 ? true : false;
		}

		public CompareFunc<string> vercmp = Alpm.pkg_vercmp;

		public HashTable<string, int64?> get_clean_cache_details (uint64 keep_nb, bool only_uninstalled) {
			var filenames_size = new HashTable<string, int64?> (str_hash, str_equal);
			var pkg_version_filenames = new HashTable<string, SList<string>> (str_hash, str_equal);
			var pkg_versions = new HashTable<string, SList<string>> (str_hash, str_equal);
			// compute all infos
			unowned Alpm.List<unowned string> cachedirs_names = alpm_handle.cachedirs;
			while (cachedirs_names != null) {
				unowned string cachedir_name = cachedirs_names.data;
				var cachedir = File.new_for_path (cachedir_name);
				try {
					FileEnumerator enumerator = cachedir.enumerate_children ("standard::*", FileQueryInfoFlags.NONE);
					FileInfo info;
					while ((info = enumerator.next_file (null)) != null) {
						unowned string filename = info.get_name ();
						string absolute_filename = "%s%s".printf (cachedir_name, filename);
						string name_version_release = filename.slice (0, filename.last_index_of_char ('-'));
						int release_index = name_version_release.last_index_of_char ('-');
						string name_version = name_version_release.slice (0, release_index);
						int version_index = name_version.last_index_of_char ('-');
						string name = name_version.slice (0, version_index);
						if (only_uninstalled && is_installed_pkg (name)) {
							continue;
						}
						filenames_size.insert (absolute_filename, info.get_size ());
						if (pkg_versions.contains (name)) {
							if (pkg_version_filenames.contains (name_version_release)) {
								// case of .sig file
								unowned SList<string> filenames = pkg_version_filenames.lookup (name_version_release);
								filenames.append (absolute_filename);
							} else {
								unowned SList<string> versions = pkg_versions.lookup (name);
								string version = name_version.slice (version_index + 1, name_version.length);
								string release = name_version_release.slice (release_index + 1, name_version_release.length);
								string version_release = "%s-%s".printf (version, release);
								versions.append ((owned) version_release);
								var filenames = new SList<string> ();
								filenames.append (absolute_filename);
								pkg_version_filenames.insert (name_version_release, (owned) filenames);
							}
						} else {
							var versions = new SList<string> ();
							string version = name_version.slice (version_index + 1, name_version.length);
							string release = name_version_release.slice (release_index + 1, name_version_release.length);
							string version_release = "%s-%s".printf (version, release);
							versions.append ((owned) version_release);
							pkg_versions.insert (name, (owned) versions);
							var filenames = new SList<string> ();
							filenames.append (absolute_filename);
							pkg_version_filenames.insert (name_version_release, (owned) filenames);
						}
					}
				} catch (GLib.Error e) {
					stderr.printf ("Error: %s\n", e.message);
				}
				cachedirs_names.next ();
			}
			if (keep_nb == 0) {
				return filenames_size;
			}
			// filter candidates
			var iter = HashTableIter<string, SList<string>> (pkg_versions);
			unowned string name;
			unowned SList<string> versions;
			while (iter.next (out name, out versions)) {
				// sort versions
				uint length = versions.length ();
				if (length > keep_nb) {
					versions.sort ((version1, version2) => {
						// reverse version 1 and version2 to have higher versions first
						return Alpm.pkg_vercmp (version2, version1);
					});
				}
				uint i = 1;
				foreach (unowned string version in versions) {
					unowned SList<string>? filenames = pkg_version_filenames.lookup ("%s-%s".printf (name, version));
					if (filenames != null) {
						foreach (unowned string filename in filenames) {
							filenames_size.remove (filename);
						}
					}
					i++;
					if (i > keep_nb) {
						break;
					}
				}
			}
			return filenames_size;
		}

		void enumerate_directory (string directory_path, ref HashTable<string, int64?> filenames_size) {
			var directory = GLib.File.new_for_path (directory_path);
			try {
				FileEnumerator enumerator = directory.enumerate_children ("standard::*", FileQueryInfoFlags.NONE);
				FileInfo info;
				while ((info = enumerator.next_file (null)) != null) {
					string absolute_filename = Path.build_path ("/", directory.get_path (), info.get_name ());
					if (info.get_file_type () == FileType.DIRECTORY) {
						enumerate_directory (absolute_filename, ref filenames_size);
					} else {
						filenames_size.insert (absolute_filename, info.get_size ());
					}
				}
			} catch (GLib.Error e) {
				stderr.printf ("Error: %s\n", e.message);
			}
		}

		public HashTable<string, int64?> get_build_files_details (string aur_build_dir) {
			var filenames_size = new HashTable<string, int64?> (str_hash, str_equal);
			enumerate_directory (aur_build_dir, ref filenames_size);
			return filenames_size;
		}

		public List<string> get_ignorepkgs () {
			var result = new List<string> ();
			unowned Alpm.List<unowned string> ignorepkgs = alpm_handle.ignorepkgs;
			while (ignorepkgs != null) {
				unowned string ignorepkg = ignorepkgs.data;
				result.append (ignorepkg);
				ignorepkgs.next ();
			}
			return result;
		}

		public bool is_installed_pkg (string pkgname) {
			return alpm_handle.localdb.get_pkg (pkgname) != null;
		}

		public Package get_installed_pkg (string pkgname) {
			return new Package.from_struct (initialise_pkg_struct (alpm_handle.localdb.get_pkg (pkgname)));
		}

		public Package find_installed_satisfier (string depstring) {
			return new Package.from_struct (initialise_pkg_struct (Alpm.find_satisfier (alpm_handle.localdb.pkgcache, depstring)));
		}

		public bool should_hold (string pkgname) {
			if (alpm_config.get_holdpkgs ().find_custom (pkgname, strcmp) != null) {
				return true;
			}
			return false;
		}

		public uint get_pkg_reason (string pkgname) {
			unowned Alpm.Package? pkg = alpm_handle.localdb.get_pkg (pkgname);
			if (pkg != null) {
				return pkg.reason;
			}
			return 0;
		}

		public List<string> get_uninstalled_optdeps (string pkgname) {
			var optdeps = new List<string> ();
			unowned Alpm.Package? pkg = get_syncpkg (pkgname);
			if (pkg != null) {
				unowned Alpm.List<unowned Alpm.Depend> optdepends = pkg.optdepends;
				while (optdepends != null) {
					string optdep = optdepends.data.compute_string ();
					unowned Alpm.Package? satisfier = Alpm.find_satisfier (alpm_handle.localdb.pkgcache, optdep);
					if (satisfier == null) {
						optdeps.append ((owned) optdep);
					}
					optdepends.next ();
				}
			}
			return optdeps;
		}

		string get_localized_string (HashTable<string,string> hashtable) {
			unowned string val;
			if (!hashtable.lookup_extended (locale, null, out val)) {
				// try with just the language
				if (!hashtable.lookup_extended (locale.split ("_")[0], null, out val)) {
					// try C locale
					if (!hashtable.lookup_extended ("C", null, out val)) {
						return "";
					}
				}
			}
			return val;
		}

		string get_app_name (As.App app) {
			return get_localized_string (app.get_names ());
		}

		string get_app_summary (As.App app) {
			return get_localized_string (app.get_comments ());
		}

		string get_app_description (As.App app) {
			return get_localized_string (app.get_descriptions ());
		}

		string get_app_icon (As.App app, string dbname) {
			string icon = "";
			app.get_icons ().foreach ((as_icon) => {
				if (as_icon.get_kind () == As.IconKind.CACHED) {
					if (as_icon.get_height () == 64) {
						icon = "/usr/share/app-info/icons/archlinux-arch-%s/64x64/%s".printf (dbname, as_icon.get_name ());
					}
				}
			});
			return icon;
		}

		string get_app_screenshot (As.App app) {
			string screenshot = "";
			app.get_screenshots ().foreach ((as_screenshot) => {
				if (as_screenshot.get_kind () == As.ScreenshotKind.DEFAULT) {
					As.Image? as_image = as_screenshot.get_source ();
					if (as_image != null) {
						unowned string? url = as_image.get_url ();
						if (url != null) {
							screenshot = url;
						}
					}
				}
			});
			return screenshot;
		}

		SList<As.App> get_pkgname_matching_apps (string pkgname) {
			var matching_apps = new SList<As.App> ();
			app_store.get_apps ().foreach ((app) => {
				if (app.get_pkgname_default () == pkgname) {
					matching_apps.append (app);
				}
			});
			return (owned) matching_apps;
		}

		PackageStruct initialise_pkg_struct (Alpm.Package? alpm_pkg) {
			if (alpm_pkg != null) {
				string installed_version = "";
				string repo_name = "";
				string desc = alpm_pkg.desc ?? "";
				string icon = "";
				string app_name = "";
				if (alpm_pkg.origin == Alpm.Package.From.LOCALDB) {
					installed_version = alpm_pkg.version;
					unowned Alpm.Package? sync_pkg = get_syncpkg (alpm_pkg.name);
					if (sync_pkg != null) {
						repo_name = sync_pkg.db.name;
					} else if (config.enable_aur) {
						var loop = new MainLoop ();
						get_aur_pkg.begin (alpm_pkg.name, (obj, res) => {
							var aur_pkg = get_aur_pkg.end (res);
							if (aur_pkg.name != "") {
								repo_name = dgettext (null, "AUR");
							}
							loop.quit ();
						});
						loop.run ();
					}
				} else if (alpm_pkg.origin == Alpm.Package.From.SYNCDB) {
					unowned Alpm.Package? local_pkg = alpm_handle.localdb.get_pkg (alpm_pkg.name);
					if (local_pkg != null) {
						installed_version = local_pkg.version;
					}
					repo_name = alpm_pkg.db.name;
				}
				if (repo_name != "" && repo_name != dgettext (null, "AUR")) {
					// find if pkgname provides only one app
					var matching_apps = get_pkgname_matching_apps (alpm_pkg.name);
					if (matching_apps.length () == 1) {
						As.App app = matching_apps.nth_data (0);
						app_name = get_app_name (app);
						desc = get_app_summary (app);
						icon = get_app_icon (app, repo_name);
					}
				}
				return PackageStruct () {
					name = alpm_pkg.name,
					app_name = (owned) app_name,
					version = alpm_pkg.version,
					installed_version = (owned) installed_version,
					desc = (owned) desc,
					repo = (owned) repo_name,
					size = alpm_pkg.isize,
					download_size = alpm_pkg.download_size,
					icon = (owned) icon
				};
			} else {
				return PackageStruct () {
					name = "",
					app_name = "",
					version = "",
					installed_version = "",
					desc = "",
					repo = "",
					icon = ""
				};
			}
		}

		List<Package> initialise_pkgs (Alpm.Package? alpm_pkg) {
			var pkgs = new List<Package> ();
			if (alpm_pkg != null) {
				string installed_version = "";
				string repo_name = "";
				if (alpm_pkg.origin == Alpm.Package.From.LOCALDB) {
					installed_version = alpm_pkg.version;
					unowned Alpm.Package? sync_pkg = get_syncpkg (alpm_pkg.name);
					if (sync_pkg != null) {
						repo_name = sync_pkg.db.name;
					} else if (config.enable_aur) {
						var loop = new MainLoop ();
						get_aur_pkg.begin (alpm_pkg.name, (obj, res) => {
							var aur_pkg = get_aur_pkg.end (res);
							if (aur_pkg.name != "") {
								repo_name = dgettext (null, "AUR");
							}
							loop.quit ();
						});
						loop.run ();
					}
				} else if (alpm_pkg.origin == Alpm.Package.From.SYNCDB) {
					unowned Alpm.Package? local_pkg = alpm_handle.localdb.get_pkg (alpm_pkg.name);
					if (local_pkg != null) {
						installed_version = local_pkg.version;
					}
					repo_name = alpm_pkg.db.name;
				}
				if (repo_name != "" && repo_name != dgettext (null, "AUR")) {
					var apps = get_pkgname_matching_apps (alpm_pkg.name);
					if (apps.length () > 0) {
						// alpm_pkg provide some apps
						foreach (unowned As.App app in apps) {
							pkgs.append (new Package.from_struct (PackageStruct () {
								name = alpm_pkg.name,
								app_name = get_app_name (app),
								version = alpm_pkg.version,
								installed_version = installed_version,
								desc = get_app_summary (app),
								repo = repo_name,
								size = alpm_pkg.isize,
								download_size = alpm_pkg.download_size,
								icon = get_app_icon (app, repo_name)
							}));
						}
					} else {
						pkgs.append (new Package.from_struct (PackageStruct () {
							name = alpm_pkg.name,
							app_name = "",
							version = alpm_pkg.version,
							installed_version = (owned) installed_version,
							desc = alpm_pkg.desc ?? "",
							repo = (owned) repo_name,
							size = alpm_pkg.isize,
							download_size = alpm_pkg.download_size,
							icon = ""
						}));
					}
				} else {
					pkgs.append (new Package.from_struct (PackageStruct () {
						name = alpm_pkg.name,
						app_name = "",
						version = alpm_pkg.version,
						installed_version = (owned) installed_version,
						desc = alpm_pkg.desc ?? "",
						repo = (owned) repo_name,
						size = alpm_pkg.isize,
						download_size = alpm_pkg.download_size,
						icon = ""
					}));
				}
			}
			return pkgs;
		}

		public List<Package> get_installed_pkgs () {
			var pkgs = new List<Package> ();
			unowned Alpm.List<unowned Alpm.Package> pkgcache = alpm_handle.localdb.pkgcache;
			while (pkgcache != null) {
				unowned Alpm.Package alpm_pkg = pkgcache.data;
				foreach (unowned Package pkg in initialise_pkgs (alpm_pkg)) {
					pkgs.append (pkg);
				}
				pkgcache.next ();
			}
			return pkgs;
		}

		public List<Package> get_installed_apps () {
			var result = new List<Package> ();
			app_store.get_apps ().foreach ((app) => {
				unowned string pkgname = app.get_pkgname_default ();
				unowned Alpm.Package? local_pkg = alpm_handle.localdb.get_pkg (pkgname);
				if (local_pkg != null) {
					unowned Alpm.Package? sync_pkg = get_syncpkg (pkgname);
					if (sync_pkg != null) {
						result.append (new Package.from_struct (PackageStruct () {
							name = sync_pkg.name,
							app_name = get_app_name (app),
							version = sync_pkg.version,
							installed_version = local_pkg.version,
							desc = get_app_summary (app),
							repo = sync_pkg.db.name,
							size = sync_pkg.isize,
							download_size = sync_pkg.download_size,
							icon = get_app_icon (app, sync_pkg.db.name)
						}));
					}
				}
			});
			return (owned) result;
		}

		public List<Package> get_explicitly_installed_pkgs () {
			var pkgs = new List<Package> ();
			unowned Alpm.List<unowned Alpm.Package> pkgcache = alpm_handle.localdb.pkgcache;
			while (pkgcache != null) {
				unowned Alpm.Package alpm_pkg = pkgcache.data;
				if (alpm_pkg.reason == Alpm.Package.Reason.EXPLICIT) {
					foreach (unowned Package pkg in initialise_pkgs (alpm_pkg)) {
						pkgs.append (pkg);
					}
				}
				pkgcache.next ();
			}
			return pkgs;
		}

		public List<Package> get_foreign_pkgs () {
			var pkgs = new List<Package> ();
			unowned Alpm.List<unowned Alpm.Package> pkgcache = alpm_handle.localdb.pkgcache;
			while (pkgcache != null) {
				unowned Alpm.Package alpm_pkg = pkgcache.data;
				bool sync_found = false;
				unowned Alpm.List<unowned Alpm.DB> syncdbs = alpm_handle.syncdbs;
				while (syncdbs != null) {
					unowned Alpm.DB db = syncdbs.data;
					unowned Alpm.Package? sync_pkg = db.get_pkg (alpm_pkg.name);
					if (sync_pkg != null) {
						sync_found = true;
						break;
					}
					syncdbs.next ();
				}
				if (sync_found == false) {
					foreach (unowned Package pkg in initialise_pkgs (alpm_pkg)) {
						pkgs.append (pkg);
					}
				}
				pkgcache.next ();
			}
			return pkgs;
		}

		public List<Package> get_orphans () {
			var pkgs = new List<Package> ();
			unowned Alpm.List<unowned Alpm.Package> pkgcache = alpm_handle.localdb.pkgcache;
			while (pkgcache != null) {
				unowned Alpm.Package alpm_pkg = pkgcache.data;
				if (alpm_pkg.reason == Alpm.Package.Reason.DEPEND) {
					Alpm.List<string> requiredby = alpm_pkg.compute_requiredby ();
					if (requiredby.length == 0) {
						Alpm.List<string> optionalfor = alpm_pkg.compute_optionalfor ();
						if (optionalfor.length == 0) {
							foreach (unowned Package pkg in initialise_pkgs (alpm_pkg)) {
								pkgs.append (pkg);
							}
						} else {
							optionalfor.free_inner (GLib.free);
						}
					} else {
						requiredby.free_inner (GLib.free);
					}
				}
				pkgcache.next ();
			}
			return pkgs;
		}

		public async List<Package> get_installed_pkgs_async () {
			var pkgs = new List<Package> ();
			new Thread<int> ("get_installed_pkgs", () => {
				pkgs =  get_installed_pkgs ();
				Idle.add (get_installed_pkgs_async.callback);
				return 0;
			});
			yield;
			return (owned) pkgs;
		}

		public async List<Package> get_installed_apps_async () {
			var pkgs = new List<Package> ();
			new Thread<int> ("get_installed_apps", () => {
				pkgs =  get_installed_apps ();
				Idle.add (get_installed_apps_async.callback);
				return 0;
			});
			yield;
			return (owned) pkgs;
		}

		public async List<Package> get_explicitly_installed_pkgs_async () {
			var pkgs = new List<Package> ();
			new Thread<int> ("get_explicitly_installed_pkgs", () => {
				pkgs =  get_explicitly_installed_pkgs ();
				Idle.add (get_explicitly_installed_pkgs_async.callback);
				return 0;
			});
			yield;
			return (owned) pkgs;
		}

		public async List<Package> get_foreign_pkgs_async () {
			var pkgs = new List<Package> ();
			new Thread<int> ("get_foreign_pkgs", () => {
				pkgs =  get_foreign_pkgs ();
				Idle.add (get_foreign_pkgs_async.callback);
				return 0;
			});
			yield;
			return (owned) pkgs;
		}

		public async List<Package> get_orphans_async () {
			var pkgs = new List<Package> ();
			new Thread<int> ("get_orphans", () => {
				pkgs =  get_orphans ();
				Idle.add (get_orphans_async.callback);
				return 0;
			});
			yield;
			return (owned) pkgs;
		}

		unowned Alpm.Package? get_syncpkg (string name) {
			unowned Alpm.Package? pkg = null;
			unowned Alpm.List<unowned Alpm.DB> syncdbs = alpm_handle.syncdbs;
			while (syncdbs != null) {
				unowned Alpm.DB db = syncdbs.data;
				pkg = db.get_pkg (name);
				if (pkg != null) {
					break;
				}
				syncdbs.next ();
			}
			return pkg;
		}

		public Package get_sync_pkg (string pkgname) {
			return new Package.from_struct (initialise_pkg_struct (get_syncpkg (pkgname)));
		}

		unowned Alpm.Package? find_dbs_satisfier (string depstring) {
			unowned Alpm.Package? pkg = null;
			unowned Alpm.List<unowned Alpm.DB> syncdbs = alpm_handle.syncdbs;
			while (syncdbs != null) {
				unowned Alpm.DB db = syncdbs.data;
				pkg = Alpm.find_satisfier (db.pkgcache, depstring);
				if (pkg != null) {
					break;
				}
				syncdbs.next ();
			}
			return pkg;
		}

		public Package find_sync_satisfier (string depstring) {
			return new Package.from_struct (initialise_pkg_struct (find_dbs_satisfier (depstring)));
		}

		Alpm.List<unowned Alpm.Package> search_local_db (string search_string) {
			Alpm.List<unowned string> needles = null;
			string[] splitted = search_string.split (" ");
			foreach (unowned string part in splitted) {
				needles.add (part);
			}
			Alpm.List<unowned Alpm.Package> result = alpm_handle.localdb.search (needles);
			// search in appstream
			string[]? search_terms = As.utils_search_tokenize (search_string);
			if (search_terms != null) {
				Alpm.List<unowned Alpm.Package> appstream_result = null;
				app_store.get_apps ().foreach ((app) => {
					uint match_score = app.search_matches_all (search_terms);
					if (match_score > 0) {
						unowned string pkgname = app.get_pkgname_default ();
						unowned Alpm.Package? alpm_pkg = alpm_handle.localdb.get_pkg (pkgname);
						if (alpm_pkg != null) {
							if (appstream_result.find (alpm_pkg, (Alpm.List.CompareFunc) alpm_pkg_compare_name) == null) {
								appstream_result.add (alpm_pkg);
							}
						}
					}
				});
				result.join (appstream_result.diff (result, (Alpm.List.CompareFunc) alpm_pkg_compare_name));
			}
			// use custom sort function
			global_search_string = search_string;
			result.sort (result.length, (Alpm.List.CompareFunc) alpm_pkg_sort_search_by_relevance);
			return result;
		}

		Alpm.List<unowned Alpm.Package> search_sync_dbs (string search_string) {
			Alpm.List<unowned string> needles = null;
			string[] splitted = search_string.split (" ");
			foreach (unowned string part in splitted) {
				needles.add (part);
			}
			Alpm.List<unowned Alpm.Package> localpkgs = alpm_handle.localdb.search (needles);
			Alpm.List<unowned Alpm.Package> syncpkgs = null;
			unowned Alpm.List<unowned Alpm.DB> syncdbs = alpm_handle.syncdbs;
			while (syncdbs != null) {
				unowned Alpm.DB db = syncdbs.data;
				if (syncpkgs.length == 0) {
					syncpkgs = db.search (needles);
				} else {
					syncpkgs.join (db.search (needles).diff (syncpkgs, (Alpm.List.CompareFunc) alpm_pkg_compare_name));
				}
				syncdbs.next ();
			}
			// remove localpkgs
			Alpm.List<unowned Alpm.Package> result = syncpkgs.diff (localpkgs, (Alpm.List.CompareFunc) alpm_pkg_compare_name);
			// search in appstream
			string[]? search_terms = As.utils_search_tokenize (search_string);
			if (search_terms != null) {
				Alpm.List<unowned Alpm.Package> appstream_result = null;
				app_store.get_apps ().foreach ((app) => {
					uint match_score = app.search_matches_all (search_terms);
					if (match_score > 0) {
						unowned string pkgname = app.get_pkgname_default ();
						unowned Alpm.Package? alpm_pkg = alpm_handle.localdb.get_pkg (pkgname);
						if (alpm_pkg == null) {
							alpm_pkg = get_syncpkg (pkgname);
							if (alpm_pkg != null) {
								if (appstream_result.find (alpm_pkg, (Alpm.List.CompareFunc) alpm_pkg_compare_name) == null) {
									appstream_result.add (alpm_pkg);
								}
							}
						}
					}
				});
				result.join (appstream_result.diff (result, (Alpm.List.CompareFunc) alpm_pkg_compare_name));
			}
			// use custom sort function
			global_search_string = search_string;
			result.sort (result.length, (Alpm.List.CompareFunc) alpm_pkg_sort_search_by_relevance);
			return result;
		}

		public List<Package> search_installed_pkgs (string search_string) {
			var pkgs = new List<Package> ();
			Alpm.List<unowned Alpm.Package> alpm_pkgs = search_local_db (search_string);
			unowned Alpm.List<unowned Alpm.Package> list = alpm_pkgs;
			while (list != null) {
				unowned Alpm.Package alpm_pkg = list.data;
				foreach (unowned Package pkg in initialise_pkgs (alpm_pkg)) {
					pkgs.append (pkg);
				}
				list.next ();
			}
			return pkgs;
		}

		public async List<Package> search_installed_pkgs_async (string search_string) {
			var pkgs = new List<Package> ();
			new Thread<int> ("search_installed_pkgs", () => {
				pkgs =  search_installed_pkgs (search_string);
				Idle.add (search_installed_pkgs_async.callback);
				return 0;
			});
			yield;
			return (owned) pkgs;
		}

		public List<Package> search_repos_pkgs (string search_string) {
			var pkgs = new List<Package> ();
			Alpm.List<unowned Alpm.Package> alpm_pkgs = search_sync_dbs (search_string);
			unowned Alpm.List<unowned Alpm.Package> list = alpm_pkgs;
			while (list != null) {
				unowned Alpm.Package alpm_pkg = list.data;
				foreach (unowned Package pkg in initialise_pkgs (alpm_pkg)) {
					pkgs.append (pkg);
				}
				list.next ();
			}
			return pkgs;
		}

		public async List<Package> search_repos_pkgs_async (string search_string) {
			var pkgs = new List<Package> ();
			new Thread<int> ("search_repos_pkgs", () => {
				pkgs =  search_repos_pkgs (search_string);
				Idle.add (search_repos_pkgs_async.callback);
				return 0;
			});
			yield;
			return (owned) pkgs;
		}

		Alpm.List<unowned Alpm.Package> search_all_dbs (string search_string) {
			Alpm.List<unowned string> needles = null;
			string[] splitted = search_string.split (" ");
			foreach (unowned string part in splitted) {
				needles.add (part);
			}
			Alpm.List<unowned Alpm.Package> result = alpm_handle.localdb.search (needles);
			Alpm.List<unowned Alpm.Package> syncpkgs = null;
			unowned Alpm.List<unowned Alpm.DB> syncdbs = alpm_handle.syncdbs;
			while (syncdbs != null) {
				unowned Alpm.DB db = syncdbs.data;
				if (syncpkgs.length == 0) {
					syncpkgs = db.search (needles);
				} else {
					syncpkgs.join (db.search (needles).diff (syncpkgs, (Alpm.List.CompareFunc) alpm_pkg_compare_name));
				}
				syncdbs.next ();
			}
			result.join (syncpkgs.diff (result, (Alpm.List.CompareFunc) alpm_pkg_compare_name));
			// search in appstream
			string[]? search_terms = As.utils_search_tokenize (search_string);
			if (search_terms != null) {
				Alpm.List<unowned Alpm.Package> appstream_result = null;
				app_store.get_apps ().foreach ((app) => {
					uint match_score = app.search_matches_all (search_terms);
					if (match_score > 0) {
						unowned string pkgname = app.get_pkgname_default ();
						unowned Alpm.Package? alpm_pkg = alpm_handle.localdb.get_pkg (pkgname);
						if (alpm_pkg == null) {
							alpm_pkg = get_syncpkg (pkgname);
						}
						if (alpm_pkg != null) {
							if (appstream_result.find (alpm_pkg, (Alpm.List.CompareFunc) alpm_pkg_compare_name) == null) {
								appstream_result.add (alpm_pkg);
							}
						}
					}
				});
				result.join (appstream_result.diff (result, (Alpm.List.CompareFunc) alpm_pkg_compare_name));
			}
			// use custom sort function
			global_search_string = search_string;
			result.sort (result.length, (Alpm.List.CompareFunc) alpm_pkg_sort_search_by_relevance);
			return result;
		}

		public List<Package> search_pkgs (string search_string) {
			var pkgs = new List<Package> ();
			Alpm.List<unowned Alpm.Package> alpm_pkgs = search_all_dbs (search_string);
			unowned Alpm.List<unowned Alpm.Package> list = alpm_pkgs;
			while (list != null) {
				unowned Alpm.Package alpm_pkg = list.data;
				foreach (unowned Package pkg in initialise_pkgs (alpm_pkg)) {
					pkgs.append (pkg);
				}
				list.next ();
			}
			return pkgs;
		}

		public async List<Package> search_pkgs_async (string search_string) {
			var pkgs = new List<Package> ();
			new Thread<int> ("search_pkgs", () => {
				pkgs =  search_pkgs (search_string);
				Idle.add (search_pkgs_async.callback);
				return 0;
			});
			yield;
			return (owned) pkgs;
		}

		AURPackageStruct initialise_aur_struct (Json.Object? json_object) {
			if (json_object == null) {
				return AURPackageStruct () {
					name = "",
					version = "",
					installed_version = "",
					desc = "",
					packagebase = "",
					outofdate = ""
				};
			}
			string installed_version = "";
			unowned Alpm.Package? pkg = alpm_handle.localdb.get_pkg (json_object.get_string_member ("Name"));
			if (pkg != null) {
				installed_version = pkg.version;
			}
			// set out of date
			string outofdate = "";
			unowned Json.Node? out_node = json_object.get_member ("OutOfDate");
			if (!out_node.is_null ()) {
				var time = GLib.Time.local ((time_t) out_node.get_int ());
				outofdate = time.format ("%x");
			}
			return AURPackageStruct () {
				name = json_object.get_string_member ("Name"),
				version = json_object.get_string_member ("Version"),
				installed_version = (owned) installed_version,
				// desc can be null
				desc = json_object.get_null_member ("Description") ? "" : json_object.get_string_member ("Description"),
				popularity = json_object.get_double_member ("Popularity"),
				packagebase = json_object.get_string_member ("PackageBase"),
				outofdate = (owned) outofdate
			};
		}

		public async List<AURPackage> search_in_aur (string search_string) {
			if (!aur_search_results.contains (search_string)) {
				Json.Array pkgs = yield aur_search (search_string.split (" "));
				aur_search_results.insert (search_string, pkgs);
			}
			var result = new List<AURPackage> ();
			Json.Array aur_pkgs = aur_search_results.get (search_string);
			aur_pkgs.foreach_element ((array, index, node) => {
				Json.Object aur_pkg = node.get_object ();
				// remove results which is installed or exist in repos
				if (alpm_handle.localdb.get_pkg (aur_pkg.get_string_member ("Name")) == null
					&& get_syncpkg (aur_pkg.get_string_member ("Name")) == null) {
					result.append (new AURPackage.from_struct (initialise_aur_struct (aur_pkg)));
				}
			});
			return (owned) result;
		}

		public HashTable<string, Variant> search_files (string[] files) {
			var result = new HashTable<string, Variant> (str_hash, str_equal);
			foreach (unowned string file in files) {
				// search in localdb
				unowned Alpm.List<unowned Alpm.Package> pkgcache = alpm_handle.localdb.pkgcache;
				while (pkgcache != null) {
					unowned Alpm.Package alpm_pkg = pkgcache.data;
					string[] found_files = {};
					unowned Alpm.FileList filelist = alpm_pkg.files;
					Alpm.File* file_ptr = filelist.files;
					for (size_t i = 0; i < filelist.count; i++, file_ptr++) {
						// exclude directory name
						if (!file_ptr->name.has_suffix ("/")) {
							// adding / to compare
							var real_file_name = new StringBuilder (alpm_handle.root);
							real_file_name.append (file_ptr->name);
							if (file in real_file_name.str) {
								found_files += (owned) real_file_name.str;
							}
						}
					}
					if (found_files.length > 0) {
						result.insert (alpm_pkg.name, new Variant.strv (found_files));
					}
					pkgcache.next ();
				}
				// search in syncdbs
				unowned Alpm.List<unowned Alpm.DB> syncdbs = files_handle.syncdbs;
				while (syncdbs != null) {
					unowned Alpm.DB db = syncdbs.data;
					pkgcache = db.pkgcache;