scim

#!/usr/bin/env python2
from __future__ import print_function


from optparse import OptionParser
parser = OptionParser(usage='%prog [options]')

parser.add_option('-c', '--check', action='store_true',
	help='Quietly check if there is any inconsistency between'
		' repo/fs, indicating it by returning error code >0.')

parser.add_option('-s', '--summary', action='store_true',
	help='Dont show the diffs, just the summary of them.')
parser.add_option('-f', '--fix', action='store_true',
	help='Fix detected errors.'
		' In case of links, its the update of a source from a destination or link-dst creation.'
		' In case of catrefs (if there are differences), live-fs (dst) file will overwrite source.'
		' Metadata is restored from a list for known files, then list is updated with new/deleted ones.')
parser.add_option('-n', '--limit', action='store', type='int',
	help='Process only given number of correctable errors from top of the file.')

parser.add_option('-r', '--reverse', action='store_true', default=False,
	help='Reverse direction of a fixes, both for links and metadata.')
parser.add_option('-w', '--reverse-meta',
	dest='reverse_meta', action='store_true', default=False,
	help='Reverse direction of a metadata fixes, so live-fs ownership, modes'
		' and attributes will be applied to repository files and fixed in meta-list file.')

parser.add_option('-l', '--links', action='store_true',
	help='Perform any links-manipulations.'
		' Implied if none of --links or --meta specified explicitly.')
parser.add_option('-b', '--links-broken', action='store_true',
	help='Only process links that are "broken" in some way'
		' without any diff involved - e.g. "Not a symlink" cases. Implies --links.')
parser.add_option('--links-meta', action='store_true',
	help='Sync metadata (repo -> live-fs) for linked files. Implies --links.')
parser.add_option('-m', '--meta', action='store_true',
	help='Perform metadata manipulations.'
		' Implied if none of --links or --meta specified explicitly, except when user'
		' has no ability to manipulate file ownership and meta_list does not exists.')

parser.add_option('-p', '--path-match', metavar='path',
	help='Only process entries that match specified local path.')

parser.add_option('--links-list',
	action='store', dest='ln_list',
	type='str', metavar='NAME', default='.scim_links',
	help='Either basename (which will be searched'
		' upwards from cwd) or full path of links-list file. Default: "%default"')
parser.add_option('--links-exclude',
	action='store', dest='ln_exclude',
	type='str', metavar='NAME', default='.scim_links_exclude',
	help='Either basename or full path of links-exclude file.'
		' Should be in the same path as links-list. Default: "%default"')
parser.add_option('--meta-list',
	action='store', dest='meta_list',
	type='str', metavar='NAME', default='.scim_meta',
	help='Either basename or full path of metadata file.'
		' Should be in the same path as links-list. Default: "%default"')
parser.add_option('--meta-exclude',
	action='store', dest='meta_exclude',
	type='str', metavar='NAME', default='.scim_meta_exclude',
	help='Either basename or full path of metadata-exclude file.'
		' Should be in the same path as links-list. Default: "%default"')

parser.add_option('--cmd-diff',
	action='store', dest='cmd_diff',
	type='str', metavar='NAME', default='colordiff -uw',
	help='Command to use for generating diffs in case of a link mismatch. Default: "%default"')
parser.add_option('--numeric-ids',
	action='store_true', dest='numeric',
	help='Use numeric ids instead of user/group names.')
parser.add_option('-x', '--posix-subset', action='store_true',
	help='Dont raise errors on failures of ops like chmod/chown'
			' and try workarounds for some file creation/open flags like O_TRUNC.'
		' Intended for usage with not-quite-posix filesystems, e.g. mounted via fuse drivers.')
parser.add_option('--no-acl', action='store_true',
	help='Pretend that POSIX ACLs do not exist.'
		' Useful as every other LTS (even-numbered) release of linux kernel'
			' seem to crash on these with e.g. some NULL dereference "Oops".')

parser.add_option('--debug',
	action='store_true', dest='debug',
	help='Give extra info on whats going on.')

optz,argz = parser.parse_args()
if argz: parser.error('This command takes no arguments')
if optz.links_meta: optz.links = True


import itertools as it, operator as op, functools as ft
from subprocess import Popen, PIPE
from fgc import sh
import os, sys, errno, logging

if optz.check: log = logging.CRITICAL
elif optz.debug: log = logging.DEBUG
else: log = logging.INFO
logging.basicConfig(level=log)
log = logging.getLogger(sys.argv[0])


### Find scim-root check argz, listz presence

def merge_filter(stream):
	skip_flag = False
	for ln, line in enumerate(it.ifilter(None, it.imap(str.strip, stream))):
		if line.startswith('>>>>>>>'):
			skip_flag = False
			continue
		elif line.startswith('<<<<<<<'):
			skip_flag = True
			log.error('Merge-block detected on line {0}'.format(ln))
		if not skip_flag: yield line

def path_filter_check(pattern, path):
	if pattern is None: return True
	return sh.fn_xmatch(pattern, path)

if optz.ln_list.startswith('/'):
	scim_path, optz.ln_list = scim_path.rsplit(os.sep, 1)
else:
	scim_path = os.getcwd()
	while os.sep in scim_path and not os.path.exists(sh.join(scim_path, optz.ln_list)):
		scim_path = scim_path.rsplit(os.sep, 1)[0]
	if os.sep not in scim_path:
		parser.error('Unable to determine scim-path location')

for i in filter( lambda x: x.startswith('ln_')\
		or x.startswith('meta_'), parser.defaults ):
	src = getattr(optz, i)
	dst = sh.join(scim_path, os.path.basename(src))
	src_chk, dst_chk = it.imap(os.path.exists, (src, dst))
	if src.startswith('/') and (src_chk != dst_chk or (
			src_chk is True and not os.path.samefile(src, dst) )):
		parser.error( 'All list-files should reside in the'
			' same path: {0}, {1}: {2}'.format(scim_path, i, src) )
	else:
		if i == 'meta_list': meta_list_path = dst # special case, 'cause it gets updated on --fix
		setattr(optz, i, merge_filter(open(dst)) if os.path.exists(dst) else list())

if not optz.meta and not optz.links:
	if os.path.exists(meta_list_path): optz.meta = True
	elif os.geteuid() == 0: optz.meta = True
	else:
		try:
			from fgc.caps import get_process
			if 'cap_chown' in get_process(os.getpid()).effective: optz.meta = True
		except ImportError: pass # no support for posix caps in fgc
	optz.links = True



### Common spool
errz = 1 # because of some stupid checks below, TODO: make sense, man
errz_critical = False # should be either false or equal to errz



### Metadata check

## Following imports substitute unsupported feats with mock functions,
##  producing empty-set data on get operations and warnings instead of changes
feat_get, feat_set = lambda *a,**k: None, lambda *a,**k: log.warn(
	'Setting {0} is not supported in this fgc module build'.format(k['feat']) )
try: from fgc.strcaps import get_file as strcaps_get, set_file as strcaps_set
except ImportError:
	strcaps_get, strcaps_set = feat_get, ft.partial(feat_set, feat='posix caps')
try:
	if optz.no_acl:
		sh.acl = None # XXX
		raise ImportError
	from fgc.acl import get as acl_get, unset as acl_unset,\
		is_mode as acl_is_mode, get_mode as acl_get_mode
except ImportError:
	acl_get = acl_is_mode = feat_get
	acl_unset, acl_rebase = None, ft.partial(feat_set, feat='acls')

import re, stat
optz.meta_exclude = map(re.compile, optz.meta_exclude)
optz.reverse_meta ^= optz.reverse
_meta_updates = 0 # count of changes-to-make in scim-meta file, mostly aesthetic

## Helpers

def meta_get(path):
	fstat = os.lstat(path)
	meta = '{0}:{1}:{{0}}'.format( # mode part is acl-dependant
		fstat.st_uid if optz.numeric else sh.to_uname(fstat.st_uid),
		fstat.st_gid if optz.numeric else sh.to_gname(fstat.st_gid) )
	try: caps = strcaps_get(path)
	except OSError: caps = None # no kernel/fs support
	try:
		acls = acl_get(path, effective=False)
		if acl_is_mode(acls): raise OSError # just a mode reflection
	except OSError: acls = None # no kernel/fs support
	if caps or acls: meta += '/' + (caps.replace(' ', ';') if caps else '')
	if acls:
		mode = acl_get_mode(acls) | (fstat.st_mode & 07000)
		meta += '/' + ','.join(acls)
	else: mode = fstat.st_mode
	setgid_recurse = False
	if stat.S_ISDIR(mode) and mode & stat.S_ISGID:
		for subpath in it.ifilter(os.path.isdir, sh.crawl(path)):
			if not os.stat(subpath).st_mode & stat.S_ISGID: break
		else: setgid_recurse = True
	return meta.format('{0}{1}'.format(
		oct(stat.S_IMODE(mode)).lstrip('0'), 'r' if setgid_recurse else '' ))

def meta_set(path, meta):
	caps = acls = setgid_recurse = None
	try:
		meta, caps = meta.split('/', 1)
		caps, acls = caps.split('/', 1)
	except ValueError: pass
	uid, gid, mode = meta.split(':')
	if mode.endswith('r'):
		mode = mode[:-1]
		setgid_recurse = True
	mode = int(mode, 8)
	if setgid_recurse and not mode & stat.S_ISGID:
		raise ValueError( 'Non-setgid recursive modes'
			' are not supported (path: {0})'.format(path) )
	# Privileges are restored sequentially, any error aborts the process
	#  to avoid situations where suid bit or posix caps can be added to an
	#  unprivileged binary (because uid/gid/mode/acl restoration failed)
	try: sh.chown(path, uid, gid)
	except KeyError as err:
		raise OSError('No such id - {0}:{1} ({2}): {3}'.format(uid, gid, path, err))
	if not acls and acl_unset: acl_unset(path)
	if not setgid_recurse:
		try: sh.chmod(path, mode, dereference=False)
		except OSError as err:
			if not optz.posix_subset or err.errno != errno.EPERM: raise
	else:
		sh.chmod(path, mode, dereference=False)
		sh.chown(path, gid=gid)
		for path in it.ifilter(os.path.isdir, sh.crawl(path)):
			sh.chmod(path, stat.S_ISGID, dereference=False, merge=True)
	if acls: acl_rebase(acls, path, base=mode)
	if caps: strcaps_set(caps.replace(';', ' '), path)


def meta_read_list():
	global errz, _meta_updates
	srcz = dict() # { repo_path: metadata }

	## Straight check
	for line in optz.meta_list:
		node, meta = line.rsplit(None, 1)

		# Special check/(dirty)fix for absolute-path nodes
		if node.startswith('/'): node = node[len(scim_path)+1:]

		# Check if path should be dropped as excluded
		if not optz.limit or errz <= optz.limit:
			chk = False
			for chk in optz.meta_exclude:
				if chk.search(node):
					errz += 1
					log.error('Found metadata for excluded path {0} ({1})'.format(node, meta))
					chk = True
					_meta_updates += 1
					break
			if chk is True: continue

		# Check if entry is duplicate
		if node in srcz:
			meta_diff = meta
			if srcz[node] != meta: meta_diff += ' => {0}'.format(srcz[node])
			log.warn( 'Duplicate metadata info for path {0} ({1}),'
				' only the latest one will be used'.format(node, meta_diff) )
			_meta_updates += 1

		srcz[node] = meta

	## Reverse check
	for node in sh.crawl(scim_path, relative=True, exclude=optz.meta_exclude):
		if optz.limit and errz > optz.limit: continue

		path = sh.join(scim_path, node)
		if node in srcz: continue

		# Check if all path parents are in srcz
		seg_path = '/'
		for seg in path.split(os.sep):
			seg_path = sh.join(seg_path, seg)
			if not seg_path.startswith(scim_path): continue
			else: node = seg_path[len(scim_path)+1:]
			if node and node not in srcz:
				errz += 1
				meta = meta_get(path)
				log.warn('Not in metadata-list: {0} ({1})'.format(node, meta))
				srcz[node] = meta
				_meta_updates += 1

	return srcz


def update_meta(srcz):
	global errz, _meta_updates

	for node, meta in srcz.items():
		if optz.limit and errz > optz.limit: continue # just fill srcz set (to rebuild meta-list)

		## Make sure path passes the filtering
		if not path_filter_check(optz.path_match, node): continue

		path = sh.join(scim_path, node)
		try: fstat = meta_get(path)
		except OSError as err:
			errz += 1
			log.error('Failed to stat path {0}: {1}'.format(node, err))
			del srcz[node] # no stat - no meta
			_meta_updates += 1
			continue # no need to "fix" permissions here ;)
		else:
			if meta == fstat: continue
			errz += 1
			log.error('Invalid metadata {0}for path {1}: {2} instead of {3}'.format(*(
				('', node, fstat, meta) if not optz.reverse_meta else ('stored ', node, meta, fstat) )))
			if optz.reverse_meta:
				meta = srcz[node] = fstat
				_meta_updates += 1

		if not optz.reverse_meta and optz.fix:
			try: meta_set(path, meta)
			except OSError as err:
				log.warn('Unable to restore metadata ({0}) for path {1}: {2}'.format(meta, node, err))
			else: log.info('Fixed metadata for path {0}'.format(node))

	if optz.fix and _meta_updates:
		# Check metadata for list itself
		meta_list_path_rel = meta_list_path[len(scim_path)+1:]
		try: meta = srcz[meta_list_path_rel]
		except KeyError:
			meta = srcz[meta_list_path_rel] = '{0}:{0}:600'.format(sh.to_uname(0), sh.to_gname(0))
		open(meta_list_path, 'w').write(''.join( '{0} {1}\n'.format(path, meta)
			for path,meta in sorted(srcz.iteritems(), key=op.itemgetter(0)) ))
		meta_set(meta_list_path, meta)
		log.info( 'Updated metadata ({0} change{1})'\
			.format(_meta_updates, 's' if _meta_updates != 1 else '') )



if optz.meta:
	meta = meta_read_list()
	# Update must be delayed in case of reverse-meta check,
	#  because catrefs' permissions must be taken into account
	if not optz.reverse_meta: update_meta(meta)



### Links-check

## Helpers

from hashlib import md5

optz.cmd_diff = optz.cmd_diff.split()

def cp_data(src, dst):
	try: sh.cp_data(src, dst)
	except IOError as err:
		if not optz.posix_subset or err.errno != errno.EPERM: raise
		sh.cp_data(src, dst, trunc_call=True)

def digest_file(path, bs=8192):
	stream, hc = open(path, 'rb'), md5()
	while True:
		chunk = stream.read(bs)
		if not chunk: break
		hc.update(chunk)
	return hc.digest()

def get_diff(src, dst, safe=False):
	if safe:
		try:
			with open(src), open(dst): pass
		except (OSError, IOError) as err:
			return 'Unable to open file(s): {}'.format(err)
	if optz.reverse: src, dst = dst, src
	if digest_file(src) == digest_file(dst): return None # no difference on byte-level
	if optz.summary: return True # contents doesn't matter in this case
	try: proc = Popen(optz.cmd_diff + [src, dst], stdout=PIPE)
	except OSError as err:
		err = 'Unable to execute diff command: {}'.format(err)
		if safe: return err
		log.fatal(err)
		sys.exit(1)
	diff = proc.stdout.read()
	proc.wait()
	return diff

def show_diff(src, dst=None):
	diff = get_diff(src, dst) if dst is not None else src
	if diff and not optz.summary:
		log.warn(' <<<<<<<<< Source/Replacement diff:')
		sys.stderr.write(diff)
		log.warn(' >>>>>>>>> ')

def rel_src(src, dst):
	src, dst = map(os.path.abspath, (src, dst))
	dst = os.path.dirname(dst)
	src, dst = map(lambda x: x.split(os.sep), (src, dst))
	for i in range(min(len(dst), len(src))):
		if dst[i] != src[i]: break
		else: i +=1
	return os.path.join(*([os.pardir] * (len(dst)-i) + src[i:]))

def optional_reverse(src, dst, flag=optz.reverse):
	return (src, dst) if not flag else (dst, src)

T_SYM = 1 # symlink
T_RSYM = T_SYM | 2 # relative symlink
T_CR = 4 # catref
ops = {T_SYM: '->', T_RSYM: '+>', T_CR: '>'}


def update_links():
	## Init
	global errz, errz_critical
	global scim_path # can change in case it's in links itself
	srcz = dict() # filled during straight check to be used in reverse check

	## Backups dir
	if optz.fix:
		from tempfile import mkdtemp
		bak_path = mkdtemp(prefix='scim-lnck.')

	## Straight check (all linkz are correct)
	for line in optz.ln_list:

		## Critical error bail-out
		if errz_critical is not False and errz > errz_critical:
			log.fatal('Critical error, further checks skipped')
			sys.exit(1)
		else: errz_critical = False

		## Get src, type, dst
		for ltype, aop in ops.iteritems():
			try: src_rel, dst = line.split(' {0} '.format(aop))
			except ValueError: pass
			else: break
		else:
			log.error('Wrong link format: {0}'.format(line))
			log.warn('Uncorrectable error!')
			continue

		## Expand relative paths
		src = sh.join(scim_path, src_rel)
		dst = sh.join(scim_path, os.path.expanduser(dst))
		if os.path.abspath(src) == os.path.abspath(scim_path):
			scim_path = dst # rebase on symlinked path
			errz_critical = errz # error here will lead to errors everywhere
		else:
			if os.path.isdir(src):
				log.debug('Linked whole path (bad practice): {0}'.format(src))
			srcz[src] = src_rel

		if optz.limit and errz > optz.limit: continue # just fill srcz set

		## Make sure path passes the filtering
		if not path_filter_check(optz.path_match, src_rel): continue

		## Check symlink / cat reference

		# Check link source
		if not os.path.exists(src):
			errz += 1
			log.error("Source doesn't exists: {0}".format(src))
			log.warn('Uncorrectable error!')
			continue

		# Common check if dst exists at all
		if not os.path.lexists(dst):
			errz += 1
			log.error('No link destination: {0} (source: {1})'.format(dst, src))
			if optz.fix:
				try:
					if ltype & T_SYM:
						if ltype == T_RSYM: src = rel_src(src, dst)
						sh.ln(src, dst)
					elif ltype & T_CR:
						# Note that cp_p is non-recursive: cat references to dirs aren't allowed
						sh.cp(src, dst, attrs=optz.meta) # and there's no "reverse" if there's no dst
				except (sh.Error, IOError, OSError) as err:
					log.warn('Unable to create link destination ({0}): {1}'.format(dst, err))
				else: log.info('Corrected: {0} {1} {2}'.format(src, aop, dst))
			continue # other checks need dst, and fix here should restore everything else as well (cp_p or just link)

		# Check if dst is the right type (symlink / reg file)
		if (ltype & T_SYM and not os.path.islink(dst)) or (ltype & T_CR and os.path.islink(dst)):
			if ltype & T_SYM:
				if optz.links_broken and get_diff(src, dst, safe=True): continue
				log.error('Not a symlink: {}'.format(dst))
				errz += 1
			elif ltype & T_CR:
				log.error('Symlink in place of catref: {}'.format(dst))
				errz += 1
			if not optz.fix:
				if not ltype & T_CR: show_diff(src, dst)
			else:
				sh.cp(src, bak_path, attrs=optz.meta)
				try:
					if not os.path.samefile(dst, src): cp_data(*optional_reverse(dst, src))
				except OSError: pass # broken symlink
				sh.rm(dst)
				if ltype == T_RSYM: src = rel_src(src, dst)
				if ltype & T_SYM: sh.ln(src, dst)
				elif ltype & T_CR: sh.cp(src, dst, attrs=optz.meta)
				log.info('Corrected: {0} {1} {2} {3}'.format(src, '<>'[int(optz.reverse)], aop, dst))
			continue # rest of the checks will be bogus anyway

		# Check target (symlink)
		if ltype & T_SYM:
			if ltype == T_RSYM:
				src = rel_src(src, dst)
				dst_tgt = [os.readlink(dst)]
			else:
				dst_tgt = [os.readlink(dst), os.path.realpath(dst)]
			if src not in dst_tgt:
				errz += 1
				log.error('Wrong symlink: {0} -> {1}, instead of {2}'.format(dst, ' / '.join(dst_tgt), src))
				if optz.fix:
					sh.rm(dst)
					sh.ln(src, dst)
					log.info('Corrected: {0} -> {1}'.format(src, dst))

		# Check permissions and content (catref)
		elif ltype & T_CR:
			if optz.meta or optz.links_meta:
				src_meta = (meta.get(src_rel) or meta_get(src))\
					if not optz.reverse_meta else meta_get(src)
				src_meta, dst_meta = optional_reverse(src_meta, meta_get(dst), flag=optz.reverse_meta)
				if dst_meta != src_meta:
					errz += 1
					tgt = dst if not optz.reverse_meta else src
					log.error('Invalid metadata for {0}path {1}: {2} instead of {3}'.format(*(
						('live-fs ', tgt, dst_meta, src_meta) if not optz.reverse_meta
						else ('', src_rel, dst_meta, src_meta) )))
					if optz.fix:
						try: sh.cp_meta(*optional_reverse(src, dst, flag=optz.reverse_meta), attrs=True)
						except OSError, err: log.error('Unable to fix metadata on {0}: {1}'.format(tgt, err))
						else: log.info('Corrected metadata for path: {0}'.format(tgt))
			if optz.limit and errz > optz.limit: continue
			diff = get_diff(src, dst) # content diff
			if diff and not optz.links_broken:
				errz += 1
				log.error('Different content: {0}'.format(src))
				if not optz.fix: show_diff(diff)
				else:
					cp_data(*optional_reverse(dst, src))
					log.info('Corrected: {0} {1} {2}'.format(dst, '><'[int(optz.reverse)], src))

	## Reverse check that all files are listed in linkz, incorrectable
	for node in sh.crawl(
			os.path.realpath(scim_path),
			relative=True, exclude=optz.ln_exclude ):
		if optz.limit and errz > optz.limit: continue

		path = sh.join(scim_path, node)
		if path in srcz: continue
		if not os.path.isdir(path): # check if any of path parents is in srcz
			segz = path.split(os.sep)
			seg_path = '/'
			for seg in segz:
				seg_path = sh.join(seg_path, seg)
				if seg_path in srcz: break
			else:
				errz += 1
				log.warn('Not listed in links-list: {0}'.format(path))

	## Cleanup, provided the script didn't crash earlier
	if optz.fix: sh.rr(bak_path)


if optz.links: update_links()
if optz.meta and optz.reverse_meta: update_meta(meta)

sys.exit(int(bool(errz-1)) if not optz.fix else 0)