From df4a4e3e1d9a9bf2431f3449e64a0ef355daec4c Mon Sep 17 00:00:00 2001 From: Torrie Fischer Date: Sun, 20 Aug 2023 09:29:20 +0200 Subject: [PATCH] model: rewrite to use hardlinks instead of janky symlinks --- src/mpm/commands/__init__.py | 10 +++- src/mpm/commands/repo.py | 11 +++- src/mpm/commands/server.py | 83 ++++++++++++--------------- src/mpm/model.py | 20 +++++-- src/mpm/repo.py | 78 ++------------------------ src/mpm/server.py | 105 +++++++++++++++-------------------- 6 files changed, 122 insertions(+), 185 deletions(-) diff --git a/src/mpm/commands/__init__.py b/src/mpm/commands/__init__.py index 3e6824b..9f1e335 100644 --- a/src/mpm/commands/__init__.py +++ b/src/mpm/commands/__init__.py @@ -1,12 +1,18 @@ from . import server, repo, config from mpm.config import Config import click +import logging @click.group() -@click.option("--config", type=click.Path(readable=True, writable=False, +@click.option("--config", '-c', type=click.Path(readable=True, writable=False, dir_okay=False)) +@click.option("--verbose", "-v", is_flag=True, default=False) @click.pass_context -def mpm(ctx, config): +def mpm(ctx, config, verbose): + if verbose: + logging.basicConfig(level=logging.DEBUG) + else: + logging.basicConfig(level=logging.INFO) ctx.ensure_object(dict) ctx.obj['config'] = Config(config) pass diff --git a/src/mpm/commands/repo.py b/src/mpm/commands/repo.py index 37b3cfd..94bfdae 100644 --- a/src/mpm/commands/repo.py +++ b/src/mpm/commands/repo.py @@ -14,6 +14,10 @@ def repo(): def list_(ctx): for repo in ctx.obj['config'].repositories(): click.echo("{} ({})".format(click.style(repo.name,bold=True), repo.path)) + if not repo.exists(): + click.echo(click.style("\tPath not found: {}".format(repo.path), + fg='red')) + continue rows = [] currentPlugin = None pluginVersions = [] @@ -51,12 +55,15 @@ def add(ctx, name, path): @repo.command('import', help='Import a plugin to a repository') @click.argument('repo', type=RepoParamType()) -@click.argument('paths', type=click.Path(exists=True, readable=True, - path_type=pathlib.Path), nargs=-1) +@click.argument('paths', type=click.Path(exists=True, readable=True), nargs=-1) def import_(repo, paths): + if not repo.exists(): + click.echo("Repository '{}' not found at {}".format(repo.name, repo.path)) + return plugins = [] searchPaths = list(paths) for path in searchPaths: + path = pathlib.Path(path) try: if path.is_dir(): searchPaths += list(path.glob("*")) diff --git a/src/mpm/commands/server.py b/src/mpm/commands/server.py index 8499e42..8794ce9 100644 --- a/src/mpm/commands/server.py +++ b/src/mpm/commands/server.py @@ -33,37 +33,38 @@ def list_(ctx, servers): if not pluginPath.exists(): click.echo(click.style("\tServer plugin directory does not exist", fg='red')) continue - outdatedLinks = [] - missing = [] installed = [] + outdated = [] + missing = [] unmanaged = [] - conflicts = [] for state in sorted(server.pluginStates(ctx.obj['config'].repositories())): - if isinstance(state, OutdatedSymlink): - outdatedLinks.append(state) + if isinstance(state, Outdated): + outdated.append(state) elif isinstance(state, Installed): installed.append(state) elif isinstance(state, MissingVersions): missing.append(state) elif isinstance(state, UnmanagedFile): unmanaged.append(state) - elif isinstance(state, SymlinkConflict): - conflicts.append(state) rows = [] for state in sorted(installed): rows.append([ state.plugin.name, str(state.plugin.versionSpec), - str(state.currentVersion), - str(state.currentVersion), + str(state.currentVersion.version), + str(state.currentVersion.version), ]) - for state in sorted(outdatedLinks): + for state in sorted(outdated): + if state.currentVersion is None: + curStr = click.style('Missing', fg='red') + else: + curStr = click.style(str(state.currentVersion.version), fg='yellow') rows.append([ state.plugin.name, click.style(str(state.wantedVersion), fg='green'), - click.style(str(state.currentVersion), fg='yellow'), - click.style(str(state.plugin.versionSpec), fg='green') + curStr, + click.style(str(state.plugin.version), fg='green') ]) for state in sorted(missing): rows.append([ @@ -79,14 +80,6 @@ def list_(ctx, servers): click.style('Unmanaged', fg='yellow', bold=True), '' ]) - for state in sorted(conflicts): - rows.append([ - state.plugin.name, - click.style(str(state.plugin.versionSpec), fg='yellow'), - click.style('Symlink Conflict', fg='red', bold=True), - click.style(str(state.plugin.versionSpec), fg='yellow'), - ]) - click.echo("\t{}.jar".format(state.plugin.name)) if len(rows) > 0: click.echo(columnar(rows, ['Plugin', 'Wanted', 'Installed', @@ -139,34 +132,37 @@ def sync(ctx, servers): allServers = ctx.obj['config'].servers() if len(servers) > 0: allServers = servers + validRepos = [] + for repo in ctx.obj['config'].repositories(): + if not repo.exists(): + click.echo("Missing repository: '{}' not found at {}".format(repo.name, repo.path)) + else: + validRepos.append(repo) for server in allServers: + if not server.exists(): + click.echo("Server '{}' not found at {}".format(server.name, + server.path)) + continue click.echo('{} ({}):'.format(server.name, server.path)) - outdatedLinks = [] - available = [] + outdated = [] missing = [] - for state in sorted(server.pluginStates(ctx.obj['config'].repositories())): - if isinstance(state, OutdatedSymlink): - outdatedLinks.append(state) - elif isinstance(state, Available): - available.append(state) + for state in sorted(server.pluginStates(validRepos)): + if isinstance(state, Outdated): + outdated.append(state) elif isinstance(state, MissingVersions): missing.append(state) rows = [] - for state in sorted(outdatedLinks): + for state in sorted(outdated): + if state.currentVersion is None: + curStr = click.style('Missing', fg='red') + else: + curStr = click.style(str(state.currentVersion.version), fg='yellow') rows.append([ state.plugin.name, click.style(str(state.wantedVersion), fg='green'), - click.style(str(state.currentVersion), fg='yellow'), - click.style(str(state.plugin.versionSpec), fg='green') - ]) - - for state in sorted(available): - rows.append([ - state.plugin.name, - click.style(str(state.wantedVersion), fg='green'), - click.style('Missing', fg='red'), + curStr, click.style(str(state.plugin.version), fg='green') ]) @@ -183,20 +179,15 @@ def sync(ctx, servers): click.echo(columnar(rows, ['Plugin', 'Wanted', 'Installed', 'Available'], no_borders=True)) - if len(outdatedLinks) > 0 or len(available) > 0: + if len(outdated): click.echo("Apply changes? [y/N]", nl=False) answer = click.getchar().lower() if answer == "y": - with click.progressbar(length=len(available)+len(outdatedLinks), + with click.progressbar(length=len(outdated), label='Synchronizing') as bar: i = 0 - for state in available: - server.installVersion(state.plugin) - server.updateSymlinkForPlugin(state.plugin, state.plugin.version) - i += 1 - bar.update(i) - for state in outdatedLinks: - server.updateSymlinkForPlugin(state.plugin, state.wantedVersion) + for state in outdated: + server.syncToVersion(state.plugin) i += 1 bar.update(i) else: diff --git a/src/mpm/model.py b/src/mpm/model.py index 71e2de9..b5d1602 100644 --- a/src/mpm/model.py +++ b/src/mpm/model.py @@ -1,4 +1,4 @@ -from semantic_version import Spec, Version +from semantic_version import SimpleSpec, Version from functools import total_ordering import os import re @@ -22,6 +22,15 @@ class Plugin: except ValueError: raise ValueError("Cannot derive semver from '{}'".format(path)) + def normalizedFilename(self): + return "{}-{}.jar".format(self.name, self.version) + + def unlink(self): + os.unlink(self.path) + + def __repr__(self): + return "Plugin({}, {} {})".format(self.path, self.name, self.version) + def __eq__(self, other): return self.name == other.name and self.version == other.version @@ -38,10 +47,13 @@ class PluginSpec: def __init__(self, name, versionSpec): self.name = name try: - self.versionSpec = Spec(str(versionSpec)) + self.versionSpec = SimpleSpec(str(versionSpec)) except ValueError: raise ValueError("Invalid version spec for plugin {}: {}".format(name, versionSpec)) + def __repr__(self): + return "PluginSpec({}, {})".format(self.name, self.versionSpec) + def __str__(self): return "{} {}".format(self.name, self.versionSpec) @@ -77,14 +89,14 @@ class UnmanagedFile(PluginState): return self.filename < other.filename return False -class OutdatedSymlink(PluginState): +class Outdated(PluginState): def __init__(self, plugin, currentVersion, wantedVersion): super().__init__(plugin) self.currentVersion = currentVersion self.wantedVersion = wantedVersion def __str__(self): - return 'OutdatedSymlink({})'.format(self.plugin) + return 'Outdated({})'.format(self.plugin) class SymlinkConflict(PluginState): pass diff --git a/src/mpm/repo.py b/src/mpm/repo.py index d719cfe..f96d163 100644 --- a/src/mpm/repo.py +++ b/src/mpm/repo.py @@ -8,6 +8,12 @@ class Repo: self.config = config self.path = config['path'] + def exists(self): + return os.path.exists(self.path) and os.path.isdir(self.path) + + def __repr__(self): + return "Repo({})".format(self.path) + def importPlugin(self, plugin): dest = "{}/{}-{}.jar".format(self.path, plugin.name, plugin.version) shutil.copyfile(plugin.path, dest) @@ -36,75 +42,3 @@ class Repo: Plugin(fullPath) except ValueError: yield fullPath - -class Server: - def __init__(self, path, pluginSet): - self.path = path - self.pluginSet = pluginSet - - def updateSymlinkForPlugin(self, plugin, version): - pluginFilename = os.path.join(self.path, 'plugins/versions/{}-{}.jar'.format(plugin.name, version)) - pluginSymlink = os.path.join(self.path, plugin.name + '.jar') - linkDst = os.path.relpath(pluginFilename, self.path) - - if os.path.lexists(pluginSymlink): - os.unlink(pluginSymlink) - os.symlink(linkDst, pluginSymlink) - - def pluginStates(self): - managedPluginFilenames = [] - for plugin in self.pluginSet: - compatibleVersions = [] - pluginLinkName = '{}.jar'.format(plugin.name) - managedPluginFilenames.append(pluginLinkName) - - if os.path.exists(os.path.join(self.path, pluginLinkName)) and not os.path.islink(os.path.join(self.path, pluginLinkName)): - yield SymlinkConflict(plugin) - continue - - for installedVersion in self.versionsForPlugin(plugin.name): - if installedVersion in plugin.versionSpec: - compatibleVersions.append(installedVersion) - - if len(compatibleVersions) == 0: - yield MissingVersions(plugin) - else: - preferredVersion = list(reversed(sorted(compatibleVersions)))[0] - currentVersion = self.currentVersionForPlugin(plugin.name) - - if currentVersion == preferredVersion: - yield Installed(plugin, currentVersion) - else: - yield OutdatedSymlink(plugin, currentVersion, preferredVersion) - - otherPlugins = os.listdir(self.path) - for pluginFile in otherPlugins: - if os.path.isfile(os.path.join(self.path, pluginFile)) and pluginFile not in managedPluginFilenames: - yield UnmanagedFile(pluginFile) - - def currentVersionForPlugin(self, pluginName): - pluginSymlink = os.path.join(self.path, pluginName + '.jar') - if not os.path.lexists(pluginSymlink): - return None - suffix = '.jar' - pluginJar = os.path.basename(os.readlink(pluginSymlink)) - jarVersion = pluginJar[len(pluginName)+1:len(pluginJar)-len(suffix)] - try: - pluginSemver = Version.coerce(jarVersion) - except ValueError: - pluginSemver = jarVersion - return pluginSemver - - def versionsForPlugin(self, pluginName): - plugins = os.listdir(os.path.join(self.path, 'plugins', 'versions')) - for pluginJar in plugins: - if pluginJar.startswith(pluginName): - prefix = pluginName + '-' - suffix = '.jar' - jarVersion = pluginJar[len(prefix):len(pluginJar)-len(suffix)] - try: - pluginSemver = Version.coerce(jarVersion) - except ValueError: - pluginSemver = jarVersion - yield pluginSemver - diff --git a/src/mpm/server.py b/src/mpm/server.py index ed92dc1..d5f712d 100644 --- a/src/mpm/server.py +++ b/src/mpm/server.py @@ -1,6 +1,11 @@ from mpm.model import * import shutil import pathlib +import logging +import os +import semantic_version + +logger = logging.getLogger('mpm.server') class Server: def __init__(self, name, config): @@ -9,6 +14,9 @@ class Server: self.path = config['path'] self.pluginPath = self.path+'/plugins' + def exists(self): + return os.path.exists(self.path) and os.path.exists(self.pluginPath) + def plugins(self): return [PluginSpec(p['name'], p.get('version', '*')) for p in self.config['plugins']] @@ -21,83 +29,62 @@ class Server: def pluginStates(self, repos): managedPluginFilenames = [] for plugin in self.plugins(): + logger.debug("Checking %r", plugin) compatibleVersions = [] - pluginLinkName = '{}.jar'.format(plugin.name) - managedPluginFilenames.append(pluginLinkName) - if os.path.exists(os.path.join(self.pluginPath, pluginLinkName)) and not os.path.islink(os.path.join(self.pluginPath, pluginLinkName)): - yield SymlinkConflict(plugin) - continue + currentVersion = self.currentVersionForPlugin(plugin.name) + logger.debug('Current version: %s', currentVersion) + if currentVersion is not None: + if plugin.versionSpec.match(currentVersion.version): + compatibleVersions.append(currentVersion) + managedPluginFilenames.append(os.path.basename(currentVersion.path)) - for installedVersion in self.versionsForPlugin(plugin.name, repos): - if installedVersion in plugin.versionSpec: - compatibleVersions.append(installedVersion) - - if len(compatibleVersions) == 0: - for repo in repos: - for repoPlugin in repo.plugins(): - if repoPlugin.name == plugin.name and repoPlugin.version in plugin.versionSpec: + for repo in repos: + for repoPlugin in repo.plugins(): + if repoPlugin.name == plugin.name: + logger.debug("%r -> %r", repoPlugin.version, plugin.versionSpec) + logger.debug(plugin.versionSpec.match(repoPlugin.version)) + if plugin.versionSpec.match(repoPlugin.version): compatibleVersions.append(repoPlugin) - if len(compatibleVersions) == 0: - yield MissingVersions(plugin) - else: - preferredVersion = list(reversed(sorted(compatibleVersions)))[0] - yield Available(preferredVersion, plugin.versionSpec) + if len(compatibleVersions) == 0: + logger.debug('No compatible versions found for %s', plugin) + yield MissingVersions(plugin) else: preferredVersion = list(reversed(sorted(compatibleVersions)))[0] - currentVersion = self.currentVersionForPlugin(plugin.name) - - if currentVersion == preferredVersion: + logger.debug('Wanted %r, found %r', plugin, preferredVersion) + if currentVersion is not None and preferredVersion == currentVersion: + logger.debug("Already installed: %r", currentVersion) yield Installed(plugin, currentVersion) else: - yield OutdatedSymlink(plugin, currentVersion, preferredVersion) + logger.debug("Update available: %r -> %r", plugin, preferredVersion) + yield Outdated(preferredVersion, currentVersion, plugin.versionSpec) otherPlugins = os.listdir(self.pluginPath) for pluginFile in otherPlugins: if os.path.isfile(os.path.join(self.pluginPath, pluginFile)) and pluginFile not in managedPluginFilenames: + logger.debug("Unmanaged file: %s", pluginFile) yield UnmanagedFile(pluginFile) def currentVersionForPlugin(self, pluginName): - pluginSymlink = os.path.join(self.pluginPath, pluginName + '.jar') - if not os.path.lexists(pluginSymlink): - return None - suffix = '.jar' - pluginJar = os.path.basename(os.readlink(pluginSymlink)) - jarVersion = pluginJar[len(pluginName)+1:len(pluginJar)-len(suffix)] - try: - pluginSemver = Version.coerce(jarVersion) - except ValueError: - pluginSemver = jarVersion - return pluginSemver + plugins = os.listdir(self.pluginPath) + for pluginJar in plugins: + if pluginJar.startswith(pluginName) and pluginJar.endswith(".jar"): + try: + return Plugin(os.path.join(self.pluginPath, pluginJar)) + except ValueError: + continue - def versionsForPlugin(self, pluginName, repos): - if os.path.exists(os.path.join(self.pluginPath, 'versions')): - plugins = os.listdir(os.path.join(self.pluginPath, 'versions')) - for pluginJar in plugins: - if pluginJar.startswith(pluginName): - prefix = pluginName + '-' - suffix = '.jar' - jarVersion = pluginJar[len(prefix):len(pluginJar)-len(suffix)] - try: - pluginSemver = Version.coerce(jarVersion) - except ValueError: - pluginSemver = jarVersion - yield pluginSemver + def syncToVersion(self, plugin): + dest = os.path.join(self.pluginPath, plugin.normalizedFilename()) + logger.debug("Syncing %s to %s", dest, plugin) + current = self.currentVersionForPlugin(plugin.name) + if current is not None: + logger.debug("Removing current version %r", current) + current.unlink() - def updateSymlinkForPlugin(self, plugin, version): - pluginFilename = os.path.join(self.pluginPath, 'versions/{}-{}.jar'.format(plugin.name, version)) - pluginSymlink = os.path.join(self.pluginPath, plugin.name + '.jar') - linkDst = os.path.relpath(pluginFilename, self.pluginPath) - - if os.path.lexists(pluginSymlink): - os.unlink(pluginSymlink) - os.symlink(linkDst, pluginSymlink) - - def installVersion(self, plugin): - if not os.path.exists(os.path.join(self.pluginPath, 'versions')): - os.path.mkdir(os.path.join(self.pluginPath, 'versions')) - dest = os.path.join(self.pluginPath, 'versions/{}-{}.jar'.format(plugin.name, plugin.version)) try: os.link(plugin.path, dest) + logger.debug("Plugin synced") except Exception: + logger.debug("Hardlink not possible. Copying instead.") shutil.copyfile(plugin.path, dest)