Compare commits

...

4 Commits

Author SHA1 Message Date
Torrie Fischer
0a2a9d5e09 add support for server templates/inheritance 2023-08-20 10:45:08 +02:00
Torrie Fischer
5e7c703cca server: add a command to lock current versions 2023-08-20 10:24:26 +02:00
Torrie Fischer
df4a4e3e1d model: rewrite to use hardlinks instead of janky symlinks 2023-08-20 09:29:20 +02:00
Torrie Fischer
4b0472cfd3 gitignore: update with build artifacts 2023-08-20 09:28:50 +02:00
8 changed files with 276 additions and 210 deletions

5
.gitignore vendored
View File

@ -1,3 +1,8 @@
*.swp
Pipfile.lock
build
dist
*.whl
.egg-info
test-jars
*.egg-info

View File

@ -1,12 +1,18 @@
from . import server, repo, config
from mpm.config import Config
import click
import logging
@click.group()
@click.option("--config", type=click.Path(readable=True, writable=False,
@click.option("--config", '-c', type=click.Path(readable=True, writable=False,
dir_okay=False))
@click.option("--verbose", "-v", is_flag=True, default=False)
@click.pass_context
def mpm(ctx, config):
def mpm(ctx, config, verbose):
if verbose:
logging.basicConfig(level=logging.DEBUG)
else:
logging.basicConfig(level=logging.INFO)
ctx.ensure_object(dict)
ctx.obj['config'] = Config(config)
pass

View File

@ -14,6 +14,10 @@ def repo():
def list_(ctx):
for repo in ctx.obj['config'].repositories():
click.echo("{} ({})".format(click.style(repo.name,bold=True), repo.path))
if not repo.exists():
click.echo(click.style("\tPath not found: {}".format(repo.path),
fg='red'))
continue
rows = []
currentPlugin = None
pluginVersions = []
@ -51,12 +55,15 @@ def add(ctx, name, path):
@repo.command('import', help='Import a plugin to a repository')
@click.argument('repo', type=RepoParamType())
@click.argument('paths', type=click.Path(exists=True, readable=True,
path_type=pathlib.Path), nargs=-1)
@click.argument('paths', type=click.Path(exists=True, readable=True), nargs=-1)
def import_(repo, paths):
if not repo.exists():
click.echo("Repository '{}' not found at {}".format(repo.name, repo.path))
return
plugins = []
searchPaths = list(paths)
for path in searchPaths:
path = pathlib.Path(path)
try:
if path.is_dir():
searchPaths += list(path.glob("*"))

View File

@ -17,6 +17,62 @@ def add(ctx, name, path):
ctx.obj['config'].save()
click.echo("Added server {} in {}".format(name, path))
@server.command(help="Save a server's current plugin list to mpm.yaml")
@click.pass_context
@click.argument('servers', type=ServerParamType(), nargs=-1)
def lock(ctx, servers):
config = ctx.obj['config']
if len(servers) == 0:
servers = config.servers()
validRepos = []
for repo in ctx.obj['config'].repositories():
if not repo.exists():
click.echo("Missing repository: '{}' not found at {}".format(repo.name, repo.path))
else:
validRepos.append(repo)
for server in servers:
click.echo('{} ({}):'.format(server.name, server.path))
savedPlugins = []
changedPlugins = []
for state in server.pluginStates(validRepos):
if isinstance(state, Installed):
savedPlugins.append(state)
elif isinstance(state, Outdated):
changedPlugins.append(state)
elif isinstance(state, MissingVersions):
changedPlugins.append(state)
rows = []
for state in savedPlugins:
newSpec = PluginSpec(state.plugin.name,
state.currentVersion.version)
if newSpec == state.plugin:
continue
rows.append([
state.plugin.name,
click.style('{} -> {}'.format(state.plugin.versionSpec,
state.currentVersion.version), fg='green'),
state.currentVersion.version
])
server.set_plugin_version(state.plugin.name, state.currentVersion.version)
for state in changedPlugins:
if state.currentVersion is not None:
rows.append([
click.style(state.plugin.name, fg='yellow'),
click.style('{} -> {}'.format(state.plugin.versionSpec,
state.currentVersion.version),
fg='yellow'),
state.currentVersion.version
])
server.set_plugin_version(state.plugin.name, state.currentVersion.version)
if len(rows) > 0:
click.echo(columnar(rows, ['Plugin', 'Wanted', 'Installed'],
no_borders=True))
config.update_server(server.name, server.config)
click.echo("Apply changes? [y/N]", nl=False)
answer = click.getchar().lower()
if answer == "y":
config.save()
@server.command('list', help='List configured servers')
@click.pass_context
@click.argument('servers', type=ServerParamType(), nargs=-1)
@ -33,37 +89,38 @@ def list_(ctx, servers):
if not pluginPath.exists():
click.echo(click.style("\tServer plugin directory does not exist", fg='red'))
continue
outdatedLinks = []
missing = []
installed = []
outdated = []
missing = []
unmanaged = []
conflicts = []
for state in sorted(server.pluginStates(ctx.obj['config'].repositories())):
if isinstance(state, OutdatedSymlink):
outdatedLinks.append(state)
if isinstance(state, Outdated):
outdated.append(state)
elif isinstance(state, Installed):
installed.append(state)
elif isinstance(state, MissingVersions):
missing.append(state)
elif isinstance(state, UnmanagedFile):
unmanaged.append(state)
elif isinstance(state, SymlinkConflict):
conflicts.append(state)
rows = []
for state in sorted(installed):
rows.append([
state.plugin.name,
str(state.plugin.versionSpec),
str(state.currentVersion),
str(state.currentVersion),
str(state.currentVersion.version),
str(state.currentVersion.version),
])
for state in sorted(outdatedLinks):
for state in sorted(outdated):
if state.currentVersion is None:
curStr = click.style('Missing', fg='red')
else:
curStr = click.style(str(state.currentVersion.version), fg='yellow')
rows.append([
state.plugin.name,
click.style(str(state.wantedVersion), fg='green'),
click.style(str(state.currentVersion), fg='yellow'),
click.style(str(state.plugin.versionSpec), fg='green')
click.style(str(state.plugin.versionSpec), fg='green'),
curStr,
click.style(str(state.wantedVersion), fg='green')
])
for state in sorted(missing):
rows.append([
@ -79,14 +136,6 @@ def list_(ctx, servers):
click.style('Unmanaged', fg='yellow', bold=True),
''
])
for state in sorted(conflicts):
rows.append([
state.plugin.name,
click.style(str(state.plugin.versionSpec), fg='yellow'),
click.style('Symlink Conflict', fg='red', bold=True),
click.style(str(state.plugin.versionSpec), fg='yellow'),
])
click.echo("\t{}.jar".format(state.plugin.name))
if len(rows) > 0:
click.echo(columnar(rows, ['Plugin', 'Wanted', 'Installed',
@ -139,35 +188,38 @@ def sync(ctx, servers):
allServers = ctx.obj['config'].servers()
if len(servers) > 0:
allServers = servers
validRepos = []
for repo in ctx.obj['config'].repositories():
if not repo.exists():
click.echo("Missing repository: '{}' not found at {}".format(repo.name, repo.path))
else:
validRepos.append(repo)
for server in allServers:
if not server.exists():
click.echo("Server '{}' not found at {}".format(server.name,
server.path))
continue
click.echo('{} ({}):'.format(server.name, server.path))
outdatedLinks = []
available = []
outdated = []
missing = []
for state in sorted(server.pluginStates(ctx.obj['config'].repositories())):
if isinstance(state, OutdatedSymlink):
outdatedLinks.append(state)
elif isinstance(state, Available):
available.append(state)
for state in sorted(server.pluginStates(validRepos)):
if isinstance(state, Outdated):
outdated.append(state)
elif isinstance(state, MissingVersions):
missing.append(state)
rows = []
for state in sorted(outdatedLinks):
for state in sorted(outdated):
if state.currentVersion is None:
curStr = click.style('Missing', fg='red')
else:
curStr = click.style(str(state.currentVersion.version), fg='yellow')
rows.append([
state.plugin.name,
click.style(str(state.wantedVersion), fg='green'),
click.style(str(state.currentVersion), fg='yellow'),
click.style(str(state.plugin.versionSpec), fg='green')
])
for state in sorted(available):
rows.append([
state.plugin.name,
click.style(str(state.wantedVersion), fg='green'),
click.style('Missing', fg='red'),
click.style(str(state.plugin.version), fg='green')
click.style(str(state.plugin.versionSpec), fg='green'),
curStr,
click.style(str(state.wantedVersion.version), fg='green')
])
click.echo("Missing plugins:")
@ -183,20 +235,15 @@ def sync(ctx, servers):
click.echo(columnar(rows, ['Plugin', 'Wanted', 'Installed',
'Available'], no_borders=True))
if len(outdatedLinks) > 0 or len(available) > 0:
if len(outdated):
click.echo("Apply changes? [y/N]", nl=False)
answer = click.getchar().lower()
if answer == "y":
with click.progressbar(length=len(available)+len(outdatedLinks),
with click.progressbar(length=len(outdated),
label='Synchronizing') as bar:
i = 0
for state in available:
server.installVersion(state.plugin)
server.updateSymlinkForPlugin(state.plugin, state.plugin.version)
i += 1
bar.update(i)
for state in outdatedLinks:
server.updateSymlinkForPlugin(state.plugin, state.wantedVersion)
for state in outdated:
server.syncToVersion(state.wantedVersion)
i += 1
bar.update(i)
else:

View File

@ -60,11 +60,33 @@ class Config():
'path': str(path)
})
def template(self, name):
return self.config['templates'].get(name, dict())
def mergeConfig(self, serverConf):
baseConf = serverConf
baseConf['plugins'] = []
mergedPlugins = {}
templates = [self.template(n) for n in serverConf['inherit']]
for template in templates:
for plugin in template.get('plugins', []):
if plugin['name'] not in mergedPlugins:
mergedPlugins[plugin['name']] = dict()
mergedPlugins[plugin['name']].update(plugin)
for plugin in serverConf['plugins']:
if plugin['name'] not in mergedPlugins:
mergedPlugins[plugin['name']] = dict()
mergedPlugins[plugin['name']].update(plugin)
for plugin in mergedPlugins:
baseConf['plugins'].append(mergedPlugins[plugin])
print(baseConf)
return baseConf
def servers(self):
return [Server(name, c) for (name, c) in self.config['servers'].items()]
return [Server(name, self.mergeConfig(c)) for (name, c) in self.config['servers'].items()]
def server(self, name):
return Server(name, self.config['servers'][name])
return Server(name, self.mergeConfig(self.config['servers'][name]))
def update_server(self, server, config):
self.config['servers'][server] = config

View File

@ -1,4 +1,4 @@
from semantic_version import Spec, Version
from semantic_version import SimpleSpec, Version
from functools import total_ordering
import os
import re
@ -7,20 +7,37 @@ version_pattern = re.compile('^(?P<name>.+)-(?P<version>(?:\.?\d+)+).+jar$')
@total_ordering
class Plugin:
def __init__(self, path):
def __init__(self, path, forceName=None, forceVersion=None):
self.path = path
pluginName = os.path.basename(path)
if forceVersion is None:
pluginMatches = version_pattern.match(pluginName)
if pluginMatches is None:
raise ValueError("Cannot derive plugin name from '{}'".format(path))
if forceName is None:
self.name = pluginMatches['name']
else:
self.name = forceName
try:
self.version = Version.coerce(pluginMatches['version'])
except ValueError:
raise ValueError("Cannot derive semver from '{}'".format(path))
else:
self.version = forceVersion
self.name = forceName
def normalizedFilename(self):
return "{}-{}.jar".format(self.name, self.version)
def unlink(self):
os.unlink(self.path)
def __repr__(self):
return "Plugin({}, {} {})".format(self.path, self.name, self.version)
def __eq__(self, other):
return self.name == other.name and self.version == other.version
@ -38,10 +55,13 @@ class PluginSpec:
def __init__(self, name, versionSpec):
self.name = name
try:
self.versionSpec = Spec(str(versionSpec))
self.versionSpec = SimpleSpec(str(versionSpec))
except ValueError:
raise ValueError("Invalid version spec for plugin {}: {}".format(name, versionSpec))
def __repr__(self):
return "PluginSpec({}, {})".format(self.name, self.versionSpec)
def __str__(self):
return "{} {}".format(self.name, self.versionSpec)
@ -77,28 +97,19 @@ class UnmanagedFile(PluginState):
return self.filename < other.filename
return False
class OutdatedSymlink(PluginState):
class Outdated(PluginState):
def __init__(self, plugin, currentVersion, wantedVersion):
super().__init__(plugin)
self.currentVersion = currentVersion
self.wantedVersion = wantedVersion
def __str__(self):
return 'OutdatedSymlink({})'.format(self.plugin)
class SymlinkConflict(PluginState):
pass
return 'Outdated({})'.format(self.plugin)
class MissingVersions(PluginState):
pass
class Available(PluginState):
def __init__(self, repoPlugin, wantedVersion):
super().__init__(repoPlugin)
self.wantedVersion = wantedVersion
def __str__(self):
return 'Available({})'.format(self.plugin)
def __init__(self, plugin, currentVersion):
super().__init__(plugin)
self.currentVersion = currentVersion
class Installed(PluginState):
def __init__(self, plugin, currentVersion):

View File

@ -8,6 +8,12 @@ class Repo:
self.config = config
self.path = config['path']
def exists(self):
return os.path.exists(self.path) and os.path.isdir(self.path)
def __repr__(self):
return "Repo({})".format(self.path)
def importPlugin(self, plugin):
dest = "{}/{}-{}.jar".format(self.path, plugin.name, plugin.version)
shutil.copyfile(plugin.path, dest)
@ -36,75 +42,3 @@ class Repo:
Plugin(fullPath)
except ValueError:
yield fullPath
class Server:
def __init__(self, path, pluginSet):
self.path = path
self.pluginSet = pluginSet
def updateSymlinkForPlugin(self, plugin, version):
pluginFilename = os.path.join(self.path, 'plugins/versions/{}-{}.jar'.format(plugin.name, version))
pluginSymlink = os.path.join(self.path, plugin.name + '.jar')
linkDst = os.path.relpath(pluginFilename, self.path)
if os.path.lexists(pluginSymlink):
os.unlink(pluginSymlink)
os.symlink(linkDst, pluginSymlink)
def pluginStates(self):
managedPluginFilenames = []
for plugin in self.pluginSet:
compatibleVersions = []
pluginLinkName = '{}.jar'.format(plugin.name)
managedPluginFilenames.append(pluginLinkName)
if os.path.exists(os.path.join(self.path, pluginLinkName)) and not os.path.islink(os.path.join(self.path, pluginLinkName)):
yield SymlinkConflict(plugin)
continue
for installedVersion in self.versionsForPlugin(plugin.name):
if installedVersion in plugin.versionSpec:
compatibleVersions.append(installedVersion)
if len(compatibleVersions) == 0:
yield MissingVersions(plugin)
else:
preferredVersion = list(reversed(sorted(compatibleVersions)))[0]
currentVersion = self.currentVersionForPlugin(plugin.name)
if currentVersion == preferredVersion:
yield Installed(plugin, currentVersion)
else:
yield OutdatedSymlink(plugin, currentVersion, preferredVersion)
otherPlugins = os.listdir(self.path)
for pluginFile in otherPlugins:
if os.path.isfile(os.path.join(self.path, pluginFile)) and pluginFile not in managedPluginFilenames:
yield UnmanagedFile(pluginFile)
def currentVersionForPlugin(self, pluginName):
pluginSymlink = os.path.join(self.path, pluginName + '.jar')
if not os.path.lexists(pluginSymlink):
return None
suffix = '.jar'
pluginJar = os.path.basename(os.readlink(pluginSymlink))
jarVersion = pluginJar[len(pluginName)+1:len(pluginJar)-len(suffix)]
try:
pluginSemver = Version.coerce(jarVersion)
except ValueError:
pluginSemver = jarVersion
return pluginSemver
def versionsForPlugin(self, pluginName):
plugins = os.listdir(os.path.join(self.path, 'plugins', 'versions'))
for pluginJar in plugins:
if pluginJar.startswith(pluginName):
prefix = pluginName + '-'
suffix = '.jar'
jarVersion = pluginJar[len(prefix):len(pluginJar)-len(suffix)]
try:
pluginSemver = Version.coerce(jarVersion)
except ValueError:
pluginSemver = jarVersion
yield pluginSemver

View File

@ -1,6 +1,33 @@
from mpm.model import *
import shutil
import pathlib
import logging
import hashlib
import os
import semantic_version
import sys
logger = logging.getLogger('mpm.server')
def digest(fd):
if sys.version_info.minor >= 11:
return hashlib.file_digest(fd, 'sha256')
else:
hasher = hashlib.sha256()
while True:
buf = fd.read(1024)
if not buf:
break
hasher.update(buf)
return hasher
class Template:
def __init__(self, name, config):
self.name = name
self.config = config
def plugins(self):
return [PluginSpec(p['name'], p.get('version', '*')) for p in self.config['plugins']]
class Server:
def __init__(self, name, config):
@ -9,6 +36,9 @@ class Server:
self.path = config['path']
self.pluginPath = self.path+'/plugins'
def exists(self):
return os.path.exists(self.path) and os.path.exists(self.pluginPath)
def plugins(self):
return [PluginSpec(p['name'], p.get('version', '*')) for p in self.config['plugins']]
@ -18,86 +48,90 @@ class Server:
raise KeyError("Cannot add plugin multiple times.")
self.config['plugins'].append({'name': pluginSpec.name, 'version': str(pluginSpec.versionSpec)})
def set_plugin_version(self, pluginName, versionSpec):
for plugin in self.config['plugins']:
if plugin['name'] == pluginName:
plugin['version'] = str(versionSpec)
def pluginStates(self, repos):
managedPluginFilenames = []
for plugin in self.plugins():
logger.debug("Checking %r", plugin)
compatibleVersions = []
pluginLinkName = '{}.jar'.format(plugin.name)
managedPluginFilenames.append(pluginLinkName)
if os.path.exists(os.path.join(self.pluginPath, pluginLinkName)) and not os.path.islink(os.path.join(self.pluginPath, pluginLinkName)):
yield SymlinkConflict(plugin)
continue
currentVersion = self.currentVersionForPlugin(plugin.name, repos)
logger.debug('Current version: %s', currentVersion)
if currentVersion is not None:
if plugin.versionSpec.match(currentVersion.version):
compatibleVersions.append(currentVersion)
managedPluginFilenames.append(os.path.basename(currentVersion.path))
for installedVersion in self.versionsForPlugin(plugin.name, repos):
if installedVersion in plugin.versionSpec:
compatibleVersions.append(installedVersion)
if len(compatibleVersions) == 0:
for repo in repos:
for repoPlugin in repo.plugins():
if repoPlugin.name == plugin.name and repoPlugin.version in plugin.versionSpec:
if repoPlugin.name == plugin.name:
logger.debug("%r -> %r", repoPlugin.version, plugin.versionSpec)
logger.debug(plugin.versionSpec.match(repoPlugin.version))
if plugin.versionSpec.match(repoPlugin.version):
compatibleVersions.append(repoPlugin)
if len(compatibleVersions) == 0:
yield MissingVersions(plugin)
logger.debug('No compatible versions found for %s', plugin)
yield MissingVersions(plugin, currentVersion)
else:
preferredVersion = list(reversed(sorted(compatibleVersions)))[0]
yield Available(preferredVersion, plugin.versionSpec)
else:
preferredVersion = list(reversed(sorted(compatibleVersions)))[0]
currentVersion = self.currentVersionForPlugin(plugin.name)
if currentVersion == preferredVersion:
logger.debug('Wanted %r, found %r', plugin, preferredVersion)
if currentVersion is not None and preferredVersion == currentVersion:
logger.debug("Already installed: %r", currentVersion)
yield Installed(plugin, currentVersion)
else:
yield OutdatedSymlink(plugin, currentVersion, preferredVersion)
logger.debug("Update available: %r -> %r", plugin, preferredVersion)
yield Outdated(plugin, currentVersion, preferredVersion)
otherPlugins = os.listdir(self.pluginPath)
for pluginFile in otherPlugins:
if os.path.isfile(os.path.join(self.pluginPath, pluginFile)) and pluginFile not in managedPluginFilenames:
logger.debug("Unmanaged file: %s", pluginFile)
yield UnmanagedFile(pluginFile)
def currentVersionForPlugin(self, pluginName):
pluginSymlink = os.path.join(self.pluginPath, pluginName + '.jar')
if not os.path.lexists(pluginSymlink):
return None
suffix = '.jar'
pluginJar = os.path.basename(os.readlink(pluginSymlink))
jarVersion = pluginJar[len(pluginName)+1:len(pluginJar)-len(suffix)]
try:
pluginSemver = Version.coerce(jarVersion)
except ValueError:
pluginSemver = jarVersion
return pluginSemver
def versionsForPlugin(self, pluginName, repos):
if os.path.exists(os.path.join(self.pluginPath, 'versions')):
plugins = os.listdir(os.path.join(self.pluginPath, 'versions'))
def currentVersionForPlugin(self, pluginName, repos=[]):
plugins = os.listdir(self.pluginPath)
for pluginJar in plugins:
if pluginJar.startswith(pluginName):
prefix = pluginName + '-'
suffix = '.jar'
jarVersion = pluginJar[len(prefix):len(pluginJar)-len(suffix)]
if pluginJar.startswith(pluginName) and pluginJar.endswith(".jar"):
if pluginJar == pluginName + ".jar":
logger.debug("Running checksum search to find version of %s", pluginJar)
pluginPath = os.path.join(self.pluginPath, pluginJar)
with open(pluginPath, 'rb') as fd:
installedHash = digest(fd)
pluginName = pluginJar.split('.jar')[0]
defaultVersion = semantic_version.Version("0.0.0")
for repo in repos:
for plugin in repo.plugins():
if plugin.name == pluginName:
with open(plugin.path, 'rb') as fd:
foundHash = digest(fd)
if foundHash.hexdigest() == installedHash.hexdigest():
return Plugin(pluginPath,
forceName=pluginName,
forceVersion = plugin.version)
return Plugin(pluginPath,
forceName=pluginName,
forceVersion=defaultVersion)
else:
try:
pluginSemver = Version.coerce(jarVersion)
return Plugin(os.path.join(self.pluginPath, pluginJar))
except ValueError:
pluginSemver = jarVersion
yield pluginSemver
continue
def updateSymlinkForPlugin(self, plugin, version):
pluginFilename = os.path.join(self.pluginPath, 'versions/{}-{}.jar'.format(plugin.name, version))
pluginSymlink = os.path.join(self.pluginPath, plugin.name + '.jar')
linkDst = os.path.relpath(pluginFilename, self.pluginPath)
def syncToVersion(self, plugin, repos=[]):
dest = os.path.join(self.pluginPath, plugin.normalizedFilename())
logger.debug("Syncing %s to %s", dest, plugin)
current = self.currentVersionForPlugin(plugin.name, repos)
if current is not None:
logger.debug("Removing current version %r", current)
current.unlink()
if os.path.lexists(pluginSymlink):
os.unlink(pluginSymlink)
os.symlink(linkDst, pluginSymlink)
def installVersion(self, plugin):
if not os.path.exists(os.path.join(self.pluginPath, 'versions')):
os.path.mkdir(os.path.join(self.pluginPath, 'versions'))
dest = os.path.join(self.pluginPath, 'versions/{}-{}.jar'.format(plugin.name, plugin.version))
try:
os.link(plugin.path, dest)
logger.debug("Plugin synced")
except Exception:
logger.debug("Hardlink not possible. Copying instead.")
shutil.copyfile(plugin.path, dest)