xUnique.py

#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
This software is licensed under the Apache 2 license, quoted below.

Copyright 2014 Xiao Wang <wangxiao8611@gmail.com, http://fclef.wordpress.com/about>

Licensed under the Apache License, Version 2.0 (the "License"); you may not
use this file except in compliance with the License. You may obtain a copy of
the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
License for the specific language governing permissions and limitations under
the License.
"""

from __future__ import unicode_literals
from __future__ import print_function
from subprocess import (check_output as sp_co, check_call as sp_cc, CalledProcessError)
from os import path, unlink, rename, listdir
from hashlib import md5 as hl_md5
from json import (loads as json_loads, dump as json_dump)
from urllib import urlretrieve
from fileinput import (input as fi_input, close as fi_close)
from re import compile as re_compile
from sys import (argv as sys_argv, getfilesystemencoding as sys_get_fs_encoding)
from collections import deque
from filecmp import cmp as filecmp_cmp
from optparse import OptionParser


md5_hex = lambda a_str: hl_md5(a_str.encode('utf-8')).hexdigest().upper()
output_u8line = lambda a_unicode: print(a_unicode.encode('utf-8'), end='')
def warning_print(*args, **kwargs):
    new_args = list(args)
    new_args[0] = '\x1B[33m{}'.format(new_args[0])
    new_args[-1] = '{}\x1B[0m'.format(new_args[-1])
    print(*new_args, **kwargs)

def success_print(*args, **kwargs):
    new_args = list(args)
    new_args[0] = '\x1B[32m{}'.format(new_args[0])
    new_args[-1] = '{}\x1B[0m'.format(new_args[-1])
    print(*new_args, **kwargs)


class XUnique(object):
    def __init__(self, target_path, verbose=False):
        # check project path
        abs_target_path = path.abspath(target_path)
        if not path.exists(abs_target_path):
            raise XUniqueExit('Path "{}" not found!'.format(abs_target_path))
        elif abs_target_path.endswith('xcodeproj'):
            self.xcodeproj_path = abs_target_path
            self.xcode_pbxproj_path = path.join(abs_target_path, 'project.pbxproj')
        elif abs_target_path.endswith('project.pbxproj'):
            self.xcode_pbxproj_path = abs_target_path
            self.xcodeproj_path = path.dirname(self.xcode_pbxproj_path)
        else:
            raise XUniqueExit("Path must be dir '.xcodeproj' or file 'project.pbxproj'")
        self.verbose = verbose
        self.vprint = print if self.verbose else lambda *a, **k: None
        self.proj_root = self.get_proj_root()
        self.proj_json = self.pbxproj_to_json()
        self.nodes = self.proj_json['objects']
        self.root_hex = self.proj_json['rootObject']
        self.root_node = self.nodes[self.root_hex]
        self.main_group_hex = self.root_node['mainGroup']
        self.__result = {}
        # initialize root content
        self.__result.update(
            {
                self.root_hex: {'path': self.proj_root,
                                'new_key': md5_hex(self.proj_root),
                                'type': self.root_node['isa']
                }
            })
        self._is_modified = False

    @property
    def is_modified(self):
        return self._is_modified

    def pbxproj_to_json(self):
        pbproj_to_json_cmd = ['plutil', '-convert', 'json', '-o', '-', self.xcode_pbxproj_path]
        try:
            json_unicode_str = sp_co(pbproj_to_json_cmd).decode(sys_get_fs_encoding())
            return json_loads(json_unicode_str)
        except CalledProcessError as cpe:
            raise XUniqueExit("""{}
Please check:
1. You have installed Xcode Command Line Tools and command 'plutil' could be found in $PATH;
2. The project file is not broken, such like merge conflicts, incomplete content due to xUnique failure. """.format(cpe.output))

    def __set_to_result(self, parent_hex, current_hex, current_path_key):
        current_node = self.nodes[current_hex]
        if isinstance(current_path_key, (list, tuple)):
            current_path = '/'.join([current_node[i] for i in current_path_key])
        elif isinstance(current_path_key, (basestring, unicode)):
            if current_path_key in current_node.keys():
                current_path = current_node[current_path_key]
            else:
                current_path = current_path_key
        else:
            raise KeyError('current_path_key must be list/tuple/string')
        cur_abs_path = '{}/{}'.format(self.__result[parent_hex]['path'], current_path)
        self.__result.update({
            current_hex: {'path': cur_abs_path,
                          'new_key': md5_hex(cur_abs_path),
                          'type': self.nodes[current_hex]['isa']
            }
        })

    def get_proj_root(self):
        '''PBXProject name,the root node'''
        pbxproject_ptn = re_compile('(?<=PBXProject ").*(?=")')
        with open(self.xcode_pbxproj_path) as pbxproj_file:
            for line in pbxproj_file:
                # project.pbxproj is an utf-8 encoded file
                line = line.decode('utf-8')
                result = pbxproject_ptn.search(line)
                if result:
                    # Backward compatibility using suffix
                    return '{}.xcodeproj'.format(result.group())
        raise XUniqueExit("File 'project.pbxproj' is broken. Cannot find PBXProject name.")

    def unique_project(self):
        """iterate all nodes in pbxproj file:

        PBXProject
        XCConfigurationList
        PBXNativeTarget
        PBXTargetDependency
        PBXContainerItemProxy
        XCBuildConfiguration
        PBX*BuildPhase
        PBXBuildFile
        PBXReferenceProxy
        PBXFileReference
        PBXGroup
        PBXVariantGroup
        """
        self.__unique_project(self.root_hex)
        if self.verbose:
            debug_result_file_path = path.join(self.xcodeproj_path, 'debug_result.json')
            with open(debug_result_file_path, 'w') as debug_result_file:
                json_dump(self.__result, debug_result_file)
            warning_print("Debug result json file has been written to '", debug_result_file_path, sep='')
        self.substitute_old_keys()


    def substitute_old_keys(self):
        self.vprint('replace UUIDs and remove unused UUIDs')
        key_ptn = re_compile('(?<=\s)([0-9A-Z]{24}|[0-9A-F]{32})(?=[\s;])')
        for line in fi_input(self.xcode_pbxproj_path, backup='.ubak', inplace=1):
            # project.pbxproj is an utf-8 encoded file
            line = line.decode('utf-8')
            key_list = key_ptn.findall(line)
            if not key_list:
                output_u8line(line)
            else:
                new_line = line
                # remove line with non-existing element
                if self.__result.get('to_be_removed') and any(
                        i for i in key_list if i in self.__result['to_be_removed']):
                    continue
                # remove incorrect entry that somehow does not exist in project node tree
                elif not all(self.__result.get(uuid) for uuid in key_list):
                    continue
                else:
                    for key in key_list:
                        new_key = self.__result[key]['new_key']
                        new_line = new_line.replace(key, new_key)
                    output_u8line(new_line)
        fi_close()
        tmp_path = self.xcode_pbxproj_path + '.ubak'
        if filecmp_cmp(self.xcode_pbxproj_path, tmp_path, shallow=False):
            unlink(self.xcode_pbxproj_path)
            rename(tmp_path, self.xcode_pbxproj_path)
            warning_print('Ignore uniquify, no changes made to "', self.xcode_pbxproj_path, sep='')
        else:
            unlink(tmp_path)
            self._is_modified = True
            success_print('Uniquify done')

    def sort_pbxproj_pl(self):
        """
        deprecated, use pure python method sort_pbxproj() below

        https://github.com/truebit/webkit/commits/master/Tools/Scripts/sort-Xcode-project-file

        my modified version which supports:
        1. sort PBXFileReference and PBXBuildFile sections
        2. avoid creating new file if no changes made
        """
        sort_script_path = path.join(path.dirname(path.abspath(__file__)), 'sort-Xcode-project-file-mod2.pl')
        if not path.exists(sort_script_path):
            self.vprint('downloading sort-Xcode-project-file')
            f_path, http_msgs = urlretrieve(
                'https://raw.github.com/truebit/webkit/master/Tools/Scripts/sort-Xcode-project-file',
                filename=sort_script_path)
            if int(http_msgs['content-length']) < 1000:  # current is 6430
                raise XUniqueExit(
                    'Cannot download script file from "https://raw.github.com/truebit/webkit/master/Tools/Scripts/sort-Xcode-project-file"')
            for line in fi_input(sort_script_path, inplace=1, backup='.sbak'):
                output_u8line(line.replace('{24}', '{32}'))
            fi_close()
            unlink(sort_script_path + '.sbak')
        self.vprint('sort project.xpbproj file')
        sp_cc(['perl', sort_script_path, self.xcode_pbxproj_path])

    def sort_pbxproj(self, sort_pbx_by_file_name = False):
        self.vprint('sort project.xpbproj file')
        lines = []
        files_start_ptn = re_compile('^(\s*)files = \(\s*$')
        files_key_ptn = re_compile('((?<=[A-Z0-9]{24} \/\* )|(?<=[A-F0-9]{32} \/\* )).+?(?= in )')
        fc_end_ptn = '\);'
        files_flag = False
        children_start_ptn = re_compile('^(\s*)children = \(\s*$')
        children_pbx_key_ptn = re_compile('((?<=[A-Z0-9]{24} \/\* )|(?<=[A-F0-9]{32} \/\* )).+?(?= \*\/)')
        child_flag = False
        pbx_start_ptn = re_compile('^.*Begin (PBXBuildFile|PBXFileReference) section.*$')
        pbx_key_ptn = re_compile('^\s+(([A-Z0-9]{24})|([A-F0-9]{32}))(?= \/\*)')
        pbx_end_ptn = ('^.*End ', ' section.*$')
        pbx_flag = False
        last_two = deque([])

        def file_dir_cmp(x, y):
            if '.' in x:
                if '.' in y:
                    return cmp(x, y)
                else:
                    return 1
            else:
                if '.' in y:
                    return -1
                else:
                    return cmp(x, y)

        for line in fi_input(self.xcode_pbxproj_path, backup='.sbak', inplace=1):
            # project.pbxproj is an utf-8 encoded file
            line = line.decode('utf-8')
            last_two.append(line)
            if len(last_two) > 2:
                last_two.popleft()
            # files search and sort
            files_match = files_start_ptn.search(line)
            if files_match:
                output_u8line(line)
                files_flag = True
                if isinstance(fc_end_ptn, unicode):
                    fc_end_ptn = re_compile(files_match.group(1) + fc_end_ptn)
            if files_flag:
                if fc_end_ptn.search(line):
                    if lines:
                        lines.sort(key=lambda file_str: files_key_ptn.search(file_str).group())
                        output_u8line(''.join(lines))
                        lines = []
                    files_flag = False
                    fc_end_ptn = '\);'
                elif files_key_ptn.search(line) and line not in lines:
                    lines.append(line)
            # children search and sort
            children_match = children_start_ptn.search(line)
            if children_match:
                output_u8line(line)
                child_flag = True
                if isinstance(fc_end_ptn, unicode):
                    fc_end_ptn = re_compile(children_match.group(1) + fc_end_ptn)
            if child_flag:
                if fc_end_ptn.search(line):
                    if lines:
                        if self.main_group_hex not in last_two[0]:
                            lines.sort(key=lambda file_str: children_pbx_key_ptn.search(file_str).group(),
                                       cmp=file_dir_cmp)
                        output_u8line(''.join(lines))
                        lines = []
                    child_flag = False
                    fc_end_ptn = '\);'
                elif children_pbx_key_ptn.search(line) and line not in lines:
                    lines.append(line)
            # PBX search and sort
            pbx_match = pbx_start_ptn.search(line)
            if pbx_match:
                output_u8line(line)
                pbx_flag = True
                if isinstance(pbx_end_ptn, tuple):
                    pbx_end_ptn = re_compile(pbx_match.group(1).join(pbx_end_ptn))
            if pbx_flag:
                if pbx_end_ptn.search(line):
                    if lines:
                        if sort_pbx_by_file_name:
                            lines.sort(key=lambda file_str: children_pbx_key_ptn.search(file_str).group())
                        else:
                            lines.sort(key=lambda file_str: pbx_key_ptn.search(file_str).group(1))
                        output_u8line(''.join(lines))
                        lines = []
                    pbx_flag = False
                    pbx_end_ptn = ('^.*End ', ' section.*')
                elif children_pbx_key_ptn.search(line) and line not in lines:
                        lines.append(line)
            # normal output
            if not (files_flag or child_flag or pbx_flag):
                output_u8line(line)
        fi_close()
        tmp_path = self.xcode_pbxproj_path + '.sbak'
        if filecmp_cmp(self.xcode_pbxproj_path, tmp_path, shallow=False):
            unlink(self.xcode_pbxproj_path)
            rename(tmp_path, self.xcode_pbxproj_path)
            warning_print('Ignore sort, no changes made to "', self.xcode_pbxproj_path, sep='')
        else:
            unlink(tmp_path)
            self._is_modified = True
            success_print('Sort done')

    def __unique_project(self, project_hex):
        '''PBXProject. It is root itself, no parents to it'''
        self.vprint('uniquify PBXProject')
        self.vprint('uniquify PBXGroup and PBXFileRef')
        self.__unique_group_or_ref(project_hex, self.main_group_hex)
        self.vprint('uniquify XCConfigurationList')
        bcl_hex = self.root_node['buildConfigurationList']
        self.__unique_build_configuration_list(project_hex, bcl_hex)
        subprojects_list = self.root_node.get('projectReferences')
        if subprojects_list:
            self.vprint('uniquify Subprojects')
            for subproject_dict in subprojects_list:
                product_group_hex = subproject_dict['ProductGroup']
                project_ref_parent_hex = subproject_dict['ProjectRef']
                self.__unique_group_or_ref(project_ref_parent_hex, product_group_hex)
        targets_list = self.root_node['targets']
        # workaround for PBXTargetDependency referring target that have not been iterated
        for target_hex in targets_list:
            cur_path_key = ('productName', 'name')
            self.__set_to_result(project_hex, target_hex, cur_path_key)
        for target_hex in targets_list:
            self.__unique_target(target_hex)

    def __unique_build_configuration_list(self, parent_hex, build_configuration_list_hex):
        '''XCConfigurationList'''
        cur_path_key = 'defaultConfigurationName'
        self.__set_to_result(parent_hex, build_configuration_list_hex, cur_path_key)
        build_configuration_list_node = self.nodes[build_configuration_list_hex]
        self.vprint('uniquify XCConfiguration')
        for build_configuration_hex in build_configuration_list_node['buildConfigurations']:
            self.__unique_build_configuration(build_configuration_list_hex, build_configuration_hex)

    def __unique_build_configuration(self, parent_hex, build_configuration_hex):
        '''XCBuildConfiguration'''
        cur_path_key = 'name'
        self.__set_to_result(parent_hex, build_configuration_hex, cur_path_key)

    def __unique_target(self, target_hex):
        '''PBXNativeTarget PBXAggregateTarget'''
        self.vprint('uniquify PBX*Target')
        current_node = self.nodes[target_hex]
        bcl_hex = current_node['buildConfigurationList']
        self.__unique_build_configuration_list(target_hex, bcl_hex)
        dependencies_list = current_node.get('dependencies')
        if dependencies_list:
            for dependency_hex in dependencies_list:
                self.__unique_target_dependency(target_hex, dependency_hex)
        build_phases_list = current_node['buildPhases']
        for build_phase_hex in build_phases_list:
            self.__unique_build_phase(target_hex, build_phase_hex)

    def __unique_target_dependency(self, parent_hex, target_dependency_hex):
        '''PBXTargetDependency'''
        target_hex = self.nodes[target_dependency_hex]['target']
        self.__set_to_result(parent_hex, target_dependency_hex, self.__result[target_hex]['path'])
        self.__unique_container_item_proxy(target_dependency_hex, self.nodes[target_dependency_hex]['targetProxy'])

    def __unique_container_item_proxy(self, parent_hex, container_item_proxy_hex):
        '''PBXContainerItemProxy'''
        self.vprint('uniquify PBXContainerItemProxy')
        self.__set_to_result(parent_hex, container_item_proxy_hex, 'remoteInfo')
        cur_path = self.__result[container_item_proxy_hex]['path']
        current_node = self.nodes[container_item_proxy_hex]
        # re-calculate remoteGlobalIDString to a new length 32 MD5 digest
        remote_global_id_hex = current_node['remoteGlobalIDString']
        if remote_global_id_hex not in self.__result.keys():
            portal_hex = current_node['containerPortal']
            portal_path = self.__result[portal_hex]['path']
            new_rg_id_path = '{}+{}'.format(cur_path, portal_path)
            self.__result.update({
                remote_global_id_hex: {'path': new_rg_id_path,
                                       'new_key': md5_hex(new_rg_id_path),
                                       'type': '{}#{}'.format(self.nodes[container_item_proxy_hex]['isa'],
                                                              'remoteGlobalIDString')
                }
            })

    def __unique_build_phase(self, parent_hex, build_phase_hex):
        '''PBXSourcesBuildPhase PBXFrameworksBuildPhase PBXResourcesBuildPhase
        PBXCopyFilesBuildPhase PBXHeadersBuildPhase PBXShellScriptBuildPhase
        '''
        self.vprint('uniquify all kinds of PBX*BuildPhase')
        current_node = self.nodes[build_phase_hex]
        # no useful key in some build phase types, use its isa value
        bp_type = current_node['isa']
        if bp_type == 'PBXShellScriptBuildPhase':
            cur_path_key = 'name'
        else:
            cur_path_key = bp_type
        self.__set_to_result(parent_hex, build_phase_hex, cur_path_key)
        self.vprint('uniquify PBXBuildFile')
        for build_file_hex in current_node['files']:
            self.__unique_build_file(build_phase_hex, build_file_hex)

    def __unique_group_or_ref(self, parent_hex, group_ref_hex):
        '''PBXFileReference PBXGroup PBXVariantGroup PBXReferenceProxy'''
        current_hex = group_ref_hex
        if self.nodes[current_hex].get('name'):
            cur_path_key = 'name'
        elif self.nodes[current_hex].get('path'):
            cur_path_key = 'path'
        else:
            # root PBXGroup has neither path nor name, give a new name 'PBXRootGroup'
            cur_path_key = 'PBXRootGroup'
        self.__set_to_result(parent_hex, current_hex, cur_path_key)
        if self.nodes[current_hex].get('children'):
            for child_hex in self.nodes[current_hex]['children']:
                self.__unique_group_or_ref(current_hex, child_hex)
        elif self.nodes[current_hex]['isa'] == 'PBXReferenceProxy':
            self.__unique_container_item_proxy(parent_hex, self.nodes[current_hex]['remoteRef'])

    def __unique_build_file(self, parent_hex, build_file_hex):
        '''PBXBuildFile'''
        current_node = self.nodes[build_file_hex]
        file_ref_hex = current_node.get('fileRef')
        if not file_ref_hex:
            self.__result.setdefault('to_be_removed', []).append(build_file_hex)
        else:
            if self.__result.get(file_ref_hex):
                cur_path_key = self.__result[file_ref_hex]['path']
                self.__set_to_result(parent_hex, build_file_hex, cur_path_key)
            else:
                self.__result.setdefault('to_be_removed', []).extend((build_file_hex, file_ref_hex))

class XUniqueExit(SystemExit):
    def __init__(self,value):
        value = "\x1B[31m{}\x1B[0m".format(value)
        super(XUniqueExit, self).__init__(value)


def main(sys_args):
    usage = "usage: %prog [-v][-u][-s][-c][-p] path/to/Project.xcodeproj"
    description = "When neither '-u' nor '-s' option exists, xUnique will invisibly add both '-u' and '-s' in arguments"
    parser = OptionParser(usage=usage, description=description)
    parser.add_option("-v", "--verbose",
                      action="store_true", dest="verbose", default=False,
                      help="output verbose messages. default is False.")
    parser.add_option("-u", "--unique", action="store_true", dest="unique_bool", default=False,
                      help="uniquify the project file. default is False.")
    parser.add_option("-s", "--sort", action="store_true", dest="sort_bool", default=False,
                      help="sort the project file. default is False.")
    parser.add_option("-c", "--combine-commit", action="store_true", dest="combine_commit", default=False,
                      help="When project file was modified, xUnique quit with non-zero status. Without this option, the status code would be zero if so. This option is usually used in Git hook to submit xUnique result combined with your original new commit.")
    parser.add_option("-p", "--sort-pbx-by-filename", action="store_true", dest="sort_pbx_fn_bool", default=False,
                      help="sort PBXFileReference and PBXBuildFile sections in project file, ordered by file name. Without this option, ordered by MD5 digest, the same as Xcode does.")
    (options, args) = parser.parse_args(sys_args[1:])
    if len(args) < 1:
        parser.print_help()
        raise XUniqueExit(
            "xUnique requires at least one positional argument: relative/absolute path to xcodeproj.")
    xcode_proj_path = args[0].decode(sys_get_fs_encoding())
    xunique = XUnique(xcode_proj_path, options.verbose)
    if not (options.unique_bool or options.sort_bool):
        print("Uniquify and Sort")
        xunique.unique_project()
        xunique.sort_pbxproj(options.sort_pbx_fn_bool)
        success_print("Uniquify and Sort done")
    else:
        if options.unique_bool:
            print('Uniquify...')
            xunique.unique_project()
        if options.sort_bool:
            print('Sort...')
            xunique.sort_pbxproj(options.sort_pbx_fn_bool)
    if options.combine_commit:
        if xunique.is_modified:
            raise XUniqueExit("File 'project.pbxproj' was modified, please add it and then commit.")
    else:
        if xunique.is_modified:
            warning_print("File 'project.pbxproj' was modified, please add it and commit again to submit xUnique result.\nNOTICE: If you want to submit xUnique result combined with original commit, use option '-c' in command.")


if __name__ == '__main__':
    main(sys_argv)