#!/usr/bin/env python
#
# wmllint -- up-convert WML and maps between versions.
#
# By Eric S. Raymond April 2007.
#
# All conversion logic for lifting WML and maps from older versions of the
# markup to newer ones should live here.  This includes resource path changes
# and renames, also map format conversions.
#
# While the script is at it, it checks for unbalanced tags and warns about them.
#
# Takes any number of directories as arguments.  Each directory is converted.
# If no directories are specified, acts on the current directory.
#
# The recommended procedure is this:
# 1. Run it with --dryrun first to see what it will do.
# 2. If the messages look good, run without --dryrun; the old content
#    will be left in backup files with a -bak extension.
# 3. Eyeball the changes with the --diff option.
# 4. Use wmlscope, with a directory list including the Wesnoth mainline WML
#    as first argument, to check that you have no unresolved references.
# 5. Test the conversion.
# 6. Use either --clean to remove the -bak files or --revert to
#    undo the conversion.
#
# This script will barf on 1.2.x maps with custom terrains.  Also, if you
# have a single subdirectory that mixes old-style and new-style
# terrain coding it might get confused.
#
# Note: You can shut wmllint up about custom terrains by having a comment
# on the same line that includes the string "wmllint: ignore".
# You can also prevent description insertions with "wmllint: no-icon".
# Finally, you can disable stack-based malformation checks with a comment
# containing "wmllint: validate-off" and re-enable with "wmllint: validate-on".

import sys, os, re, getopt, string, copy, difflib, time
from wesnoth.wmltools import *

filemoves = {
    # Older includes all previous to 1.3.1.
    "older" : (
        # File naming error made repeatedly in NR and elsewhere.
        ("human-loyalists/human-", "human-loyalists/"),
        # These are picked to cover as many as possible of the broken
        # references in UMC on the campaign server.  Some things we
        # don't try to fix include:
        # - attack/staff.png may map to one of several staves.
        # - magic.wav may map to one of several sounds depending on the unit.
        # Some other assumption sound in current UMC as of April 2007
        # but theoretically dubious are marked with *.
        ("../music/defeat.ogg",     "defeat.ogg"),
        ("../music/victory.ogg",    "victory.ogg"),
        ("AMLA_TOUGH_2",            "AMLA_TOUGH 2"),
        ("AMLA_TOUGH_3",            "AMLA_TOUGH 3"),
        ("SOUND_LIST:DAGGER_SWISH", "SOUND_LIST:SWORD_SWISH"),
        ("arrow-hit.wav",           "bow.ogg"),
        ("arrow-miss.wav",          "bow-miss.ogg"),
        ("attacks/animal-fangs.png","attacks/fangs-animal.png"),
        ("attacks/crossbow.png",    "attacks/human-crossbow.png"),        #*
        ("attacks/dagger.png",      "attacks/human-dagger.png"),          #*
        ("attacks/darkstaff.png",   "attacks/staff-necromantic.png"),
        ("attacks/human-fist.png",  "attacks/fist-human.png"),
        ("attacks/human-mace.png",  "attacks/mace.png"),
        ("attacks/human-sabre.png", "attacks/sabre-human.png"),
        ("attacks/icebolt.png",     "attacks/iceball.png"),    # Is this right?
        ("attacks/lightingbolt.png","attacks/lightning.png"),
        ("attacks/missile.png",     "attacks/magic-missile.png"),
        ("attacks/morning_star.png","attacks/morning-star.png"),
        ("attacks/plaguestaff.png", "attacks/staff-plague.png"),
        ("attacks/slam.png",        "attacks/slam-drake.png"),
        ("attacks/staff-magical.png","attacks/staff-magic.png"),
        ("attacks/sword-paladin.png","attacks/sword-holy.png"),
        ("attacks/sword.png",       "attacks/human-sword.png"),                #*
        ("attacks/sword_holy.png",  "attacks/sword-holy.png"),
        ("attacks/throwing-dagger-human.png", "attacks/dagger-thrown-human.png"),
        ("bow-hit.ogg",             "bow.ogg"),
        ("bow-hit.wav",             "bow.ogg"),
        ("bowman-attack-sword.png", "bowman-sword-1.png"),
        ("bowman-attack1.png",      "bowman-ranged-1.png"),
        ("bowman-attack2.png",      "bowman-ranged-2.png"),
        ("creepy.ogg",              "underground.ogg"),
        ("dwarves/warrior.png",     "dwarves/fighter.png"),
        ("eagle.wav",               "gryphon-shriek-1.ogg"),
        ("elfland.ogg",             "elf-land.ogg"),
        ("elvish-fighter.png",      "elves-wood/fighter.png"),
        ("elvish-hero.png",         "elves-wood/hero.png"),
        ("fist.wav",                "fist.ogg"),
        ("flame-miss.ogg",          "flame-big-miss.ogg"),
        ("flame.ogg",               "flame-big.ogg"),
        ("gameplay2.ogg",           "gameplay02.ogg"),                    # Changes in 1.3.2
        ("goblin-hit2.ogg",         "goblin-hit-2.ogg"),
        ("hatchet-miss-1.ogg",      "hatchet-miss.wav"),
        ("heal.ogg",                "heal.wav"),
        ("hiss-big.ogg",            "hiss-big.wav"),
        ("human-dagger.png",        "dagger-human.png"),
        ("human-male-die.ogg",      "human-die-1.ogg"),
        ("human-male-hit.ogg",      "human-hit-1.ogg"),
        ("human-male-weak-die.ogg", "human-old-die-1.ogg"),
        ("human-male-weak-hit.ogg", "human-old-hit-1.ogg"),
        ("human-sword.png",         "sword-human.png"),
        ("items/castle-ruins.png",  "scenery/castle-ruins.png"),
        ("items/fire.png",          "scenery/fire1.png"),
        ("items/fire1.png",         "scenery/fire1.png"),
        ("items/fire2.png",         "scenery/fire2.png"),
        ("items/fire3.png",         "scenery/fire3.png"),
        ("items/fire4.png",         "scenery/fire4.png"),
        ("items/hero-icon.png",     "misc/hero-icon.png"),
        ("items/leanto.png",        "scenery/leanto.png"),
        ("items/lighthouse.png",    "scenery/lighthouse.png"),
        ("items/monolith1.png",     "scenery/monolith1.png"),
        ("items/monolith2.png",     "scenery/monolith2.png"),
        ("items/monolith3.png",     "scenery/monolith3.png"),
        ("items/monolith4.png",     "scenery/monolith4.png"),
        ("items/ring1.png",         "items/ring-silver.png"),  # Is this right?
        ("items/ring2.png",         "items/ring-gold.png"),    # Is this right?
        ("items/rock1.png",         "scenery/rock1.png"),
        ("items/rock2.png",         "scenery/rock2.png"),
        ("items/rock3.png",         "scenery/rock3.png"),
        ("items/rock4.png",         "scenery/rock4.png"),
        ("items/signpost.png",      "scenery/signpost.png"),
        ("items/slab.png",          "scenery/slab-1.png"),
        ("items/well.png",          "scenery/well.png"),
        ("knife.ogg",               "dagger-swish.wav"),       # Is this right?
        ("knife.wav",               "dagger-swish.wav"),       # Is this right?
        ("lightning.wav",           "lightning.ogg"),
        ("longbowman-ranged-1.png", "longbowman-bow-attack1.png"),
        ("longbowman-ranged-2.png", "longbowman-bow-attack2.png"),
        ("longbowman-ranged-3.png", "longbowman-bow-attack3.png"),
        ("longbowman-ranged-4.png", "longbowman-bow-attack4.png"),
        ("misc/chest.png",          "items/chest.png"),
        ("misc/dwarven-doors.png",  "scenery/dwarven-doors-closed.png"),
        ("misc/mine.png",           "scenery/mine-abandoned.png"),
        ("misc/nest-empty.png",     "scenery/nest-empty.png"),
        ("misc/rocks.png",          "scenery/rubble.png"),
        ("misc/snowbits.png",       "scenery/snowbits.png"),
        ("misc/temple.png",         "scenery/temple1.png"),
        ("miss.wav",                "miss-1.ogg"),
        ("orc-die.wav",             "orc-die-1.ogg"),
        ("orc-hit.wav",             "orc-hit-1.ogg"),
        ("ork-die-2.ogg",           "orc-die-2.ogg"),
        ("pistol.wav",              "gunshot.wav"),
        ("spear-miss-1.ogg",        "spear-miss.ogg"),
        ("spearman-attack-south-1.png", "spearman-attack-s-1.png"),
        ("spearman-attack-south-2.png", "spearman-attack-s-2.png"),
        ("spearman-attack-south-3.png", "spearman-attack-s-3.png"),
        ("squishy-miss-1.ogg",      "squishy-miss.wav"),
        ("sword-swish.wav",         "sword-1.ogg"),
        ("sword.wav",               "sword-1.ogg"),
        ("terrain/flag-1.png",      "flags/flag-1.png"),
        ("terrain/flag-2.png",      "flags/flag-2.png"),
        ("terrain/flag-3.png",      "flags/flag-3.png"),
        ("terrain/flag-4.png",      "flags/flag-4.png"),
        ("terrain/rocks.png",       "scenery/rock2.png"),
        ("terrain/signpost.png",    "scenery/signpost.png"),
        ("terrain/village-cave-tile.png","terrain/village/cave-tile.png"),
        ("terrain/village-dwarven-tile.png","terrain/village/dwarven-tile.png"),
        ("terrain/village-elven4.png","terrain/village/elven4.png"),
        ("terrain/village-human-snow.png", "terrain/village/human-snow.png"),
        ("terrain/village-human.png","terrain/village/human.png"),
        ("terrain/village-human4.png","terrain/village/human4.png"),
        ("throwing-dagger-swish.wav","dagger-swish.wav"),      # Is this right?
        ("units/undead/ghost-attack.png", "units/undead/ghost-attack-2.png"),
        ("units/undead/ghost-attack1.png", "units/undead/ghost-attack-1.png"),
        ("wolf-attack.wav",         "wolf-bite.ogg"),
        ("wolf-cry.wav",            "wolf-die.wav"),
        ("wose-attack.wav",         "wose-attack.ogg"),
        (r"wose\.attack.ogg",       "wose-attack.ogg"),
    ),
    "1.3.1" : (
        # Peasant images moved to a new directory
        ("human-loyalists/peasant.png", "human-peasants/peasant.png"),
        ("human-loyalists/peasant-attack.png", "human-peasants/peasant-attack.png"),
        ("human-loyalists/peasant-attack2.png", "human-peasants/peasant-attack2.png"),
        ("human-loyalists/peasant-ranged.png", "human-peasants/peasant-ranged.png"),
        ("human-loyalists/peasant-idle-1.png", "human-peasants/peasant-idle-1.png"),
        ("human-loyalists/peasant-idle-2.png", "human-peasants/peasant-idle-2.png"),
        ("human-loyalists/peasant-idle-3.png", "human-peasants/peasant-idle-3.png"),
        ("human-loyalists/peasant-idle-4.png", "human-peasants/peasant-idle-4.png"),
        ("human-loyalists/peasant-idle-5.png", "human-peasants/peasant-idle-5.png"),
        ("human-loyalists/peasant-idle-6.png", "human-peasants/peasant-idle-6.png"),
        ("human-loyalists/peasant-idle-7.png", "human-peasants/peasant-idle-7.png"),
        # All Great Mage attacks were renamed
        ("great-mage-attack-magic1.png", "great-mage-attack-magic-1.png"),
        ("great-mage-attack-magic2.png", "great-mage-attack-magic-2.png"),
        ("great-mage+female-attack-magic1.png", "great-mage+female-attack-magic-1.png"),
        ("great-mage+female-attack-magic2.png", "great-mage+female-attack-magic-2.png"),
        ("great-mage-attack-staff1.png", "great-mage-attack-staff-1.png"),
        ("great-mage-attack-staff2.png", "great-mage-attack-staff-2.png"),
        ("great-mage+female-attack-staff1.png", "great-mage+female-attack-staff-1.png"),
        ("great-mage+female-attack-staff2.png", "great-mage+female-attack-staff-2.png"),
        # All Arch Mage attacks were renamed
        ("arch-mage-attack-magic1.png", "arch-mage-attack-magic-1.png"),
        ("arch-mage-attack-magic2.png", "arch-mage-attack-magic-2.png"),
        ("arch-mage+female-attack-magic1.png", "arch-mage+female-attack-magic-1.png"),
        ("arch-mage+female-attack-magic2.png", "arch-mage+female-attack-magic-2.png"),
        ("arch-mage-attack-staff1.png", "arch-mage-attack-staff-1.png"),
        ("arch-mage-attack-staff2.png", "arch-mage-attack-staff-2.png"),
        ("arch-mage+female-attack-staff1.png", "arch-mage+female-attack-staff-1.png"),
        ("arch-mage+female-attack-staff2.png", "arch-mage+female-attack-staff-2.png"),
        # All Red Mage attacks were renamed
        ("red-mage-attack-magic1.png", "red-mage-attack-magic-1.png"),
        ("red-mage-attack-magic2.png", "red-mage-attack-magic-2.png"),
        ("red-mage+female-attack-magic1.png", "red-mage+female-attack-magic-1.png"),
        ("red-mage+female-attack-magic2.png", "red-mage+female-attack-magic-2.png"),
        ("red-mage-attack-staff1.png", "red-mage-attack-staff-1.png"),
        ("red-mage-attack-staff2.png", "red-mage-attack-staff-2.png"),
        ("red-mage+female-attack-staff1.png", "red-mage+female-attack-staff-1.png"),
        ("red-mage+female-attack-staff2.png", "red-mage+female-attack-staff-2.png"),
        # Timothy Pinkham supplied titles for two of his music files.
        # Zhaytee supplied a title for wesnoth-1.ogg
        # gameplay03.ogg, and and wesnoth-[25].ogg already had titles.
        ("gameplay01.ogg", "knolls.ogg"),
        ("gameplay02.ogg", "wanderer.ogg"),
        ("gameplay03.ogg", "battle.ogg"),
        ("wesnoth-1.ogg", "revelation.ogg"),
        ("wesnoth-2.ogg", "loyalists.ogg"),
        ("wesnoth-5.ogg", "northerners.ogg"),
        # And the holy->arcane change
        ("type=holy", "type=arcane"),
        ("holy=", "arcane="),
    ),
    "1.3.2" : (
        ("misc/item-holywater.png", "items/holywater.png"),
        ("orc-small-hit.wav",       "orc-small-hit-1.ogg"),
    ),
    "1.3.3" : (
    	("sounds/dragonstick-hit.ogg", "sounds/dragonstick-hit-1.ogg"),
    	("sounds/dragonstick-miss.ogg", "sounds/dragonstick-miss.wav"),
    ),
    "1.3.4" : (
    	# This release changed from numeric to string palette IDs
        ("RC(magenta>1)", "RC(magenta>red)"),
        ("RC(magenta>2)", "RC(magenta>green)"),
        ("RC(magenta>3)", "RC(magenta>blue)"),
        ("RC(magenta>4)", "RC(magenta>purple)"),
        ("RC(magenta>5)", "RC(magenta>black)"),
        ("RC(magenta>6)", "RC(magenta>brown)"),
        ("RC(magenta>7)", "RC(magenta>orange)"),
        ("RC(magenta>8)", "RC(magenta>white)"),
        ("RC(magenta>9)", "RC(magenta>teal)"),
        ("colour=1", "colour=red"),
        ("colour=2", "colour=green"),
        ("colour=3", "colour=blue"),
        ("colour=4", "colour=purple"),
        ("colour=5", "colour=black"),
        ("colour=6", "colour=brown"),
        ("colour=7", "colour=orange"),
        ("colour=8", "colour=white"),
        ("colour=9", "colour=teal"),
    ),
    "1.3.5" : (),	# This was an aborted release
    "1.3.6" : (
	("Soul Shooter", "Banebow"),
    ),
    # An empty sentinel value at end is required.
    "future" : (),
}

# Turn all the filemove string substition pairs into nearly equivalent
# regexp-substitution pairs, forbidding the match from being preceded
# by a dash.  This prevents, e.g., "miss.ogg" false-matching on "big-miss.ogg".
for (key, value) in filemoves.items():
    filemoves[key] = map(lambda (old, new): (re.compile("(?<!-)"+old), new), value)

# 1.2.x to 1.3.2 terrain conversion
conversion1 = {
    " " : "_s",
    "&" : "Mm^Xm",
    "'" : "Uu^Ii",
    "/" : "Ww^Bw/",
    "1" : "1 _K",
    "2" : "2 _K",
    "3" : "3 _K",
    "4" : "4 _K",
    "5" : "5 _K",
    "6" : "6 _K",
    "7" : "7 _K",
    "8" : "8 _K",
    "9" : "9 _K",
    "?" : "Gg^Fet",
    "A" : "Ha^Vhha",
    "B" : "Dd^Vda",
    "C" : "Ch",
    "D" : "Uu^Vu",
    "E" : "Rd",
    "F" : "Aa^Fpa",
    "G" : "Gs",
    "H" : "Ha",
    "I" : "Dd",
    "J" : "Hd",
    "K" : "_K",
    "L" : "Gs^Vht",
    "M" : "Md",
    "N" : "Chr",
    "P" : "Dd^Do",
    "Q" : "Chw",
    "R" : "Rr",
    "S" : "Aa",
    "T" : "Gs^Ft",
    "U" : "Dd^Vdt",
    "V" : "Aa^Vha",
    "W" : "Xu",
    "X" : "Qxu",
    "Y" : "Ss^Vhs",
    "Z" : "Ww^Vm",
    "[" : "Uh",
    "\\": "Ww^Bw\\",
    "]" : "Uu^Uf",
    "a" : "Hh^Vhh",
    "b" : "Mm^Vhh",
    "c" : "Ww",
    "d" : "Ds",
    "e" : "Aa^Vea",
    "f" : "Gs^Fp",
    "g" : "Gg",
    "h" : "Hh",
    "i" : "Ai",
    "k" : "Wwf",
    "l" : "Ql",
    "m" : "Mm",
    "n" : "Ce",
    "o" : "Cud",
    "p" : "Uu^Vud",
    "q" : "Chs",
    "r" : "Re",
    "s" : "Wo",
    "t" : "Gg^Ve",
    "u" : "Uu",
    "v" : "Gg^Vh",
    "w" : "Ss",
    "|" : "Ww^Bw|",
    "~" : "_f",
}
max_len = max(*map(len, conversion1.values()))
width = max_len+2

def neighborhood(x, y, map):
    "Returns list of original location+adjacent locations from a hex map"
    odd = (x) % 2
    adj = [map[y][x]];
    if x > 0:
	adj.append(map[y][x-1])
    if x < len(map[y])-1:
	adj.append(map[y][x+1])
    if y > 0:
	adj.append(map[y-1][x])
    if y < len(map)-1:
	adj.append(map[y+1][x])
    if x > 0 and y > 0 and not odd:
	adj.append(map[y-1][x-1])
    if x < len(map[y])-1 and y > 0 and not odd:
	adj.append(map[y-1][x+1])
    if x > 0 and y < len(map)-1 and odd:
	adj.append(map[y+1][x-1])
    if x < len(map[y])-1 and y < len(map)-1 and odd:
	adj.append(map[y+1][x+1])
    return adj

def maptransform1(filename, baseline, inmap, y):
    "Transform a map line from 1.2.x to 1.3.x format."
    global lock_terrain_coding
    # The one truly ugly piece of implementation.
    # We're relying here on maps being seen before scenario files.
    # We notice whether the maps are oldstyle (single-letter codes)
    # or newstyle (multiletter comma-seeparated fields) and retain that
    # information to help with ambiguous cases later on.  We're also relying
    # on terrain coding to be consistent within a single subdirectory. 
    if len(inmap[y][0]) > 1:
        lock_terrain_coding = "newstyle"
    else:
        format = "%%%d.%ds" % (width, max_len)
        for (x, field) in enumerate(inmap[y]):
            if field in conversion1:
                lock_terrain_coding = "oldstyle"
                inmap[y][x] = format % conversion1[field]
            else:
                raise maptransform_error(filename, baseline+y+1,
                                         "unrecognized map element %s at (%s, %s)" % (`field`, x, y))

# 1.3.1 -> 1.3.2 terrain conversions
conversion2 = {
    re.compile(r"(?<!\^)Bww([|/\\])") : "Ww^Bw\\1",
    re.compile(r"(?<!\^)Bwo([|/\\])") : "Wo^Bw\\1",
    re.compile(r"(?<!\^)Bss([|/\\])") : "Ss^Bw\\1",
    re.compile(r"(?<!\^)Dc\b") : "Dd^Dc",
    re.compile(r"(?<!\^)Dr\b") : "Dd^Dr",
    re.compile(r"(?<!\^)Do\b") : "Dd^Do",
    re.compile(r"(?<!\^)Fa\b") : "Aa^Fpa",
    re.compile(r"(?<!\^)Fet\b") : "Gg^Fet",
    re.compile(r"(?<!\^)Ff\b") : "Gs^Fp",
    re.compile(r"(?<!\^)Ft\b") : "Gs^Ft",
    re.compile(r"(?<!\^)Rfvs\b") : "Re^Gvs",
    re.compile(r"(?<!\^)Uf\b") : "Uu^Uf",
    re.compile(r"(?<!\^)Uui\b") : "Uu^Ii",
    re.compile(r"(?<!\^)Uhi\b") : "Uh^Ii",
    re.compile(r"(?<!\^)Vda\b") : "Dd^Vda",
    re.compile(r"(?<!\^)Vdt\b") : "Dd^Vdt",
    re.compile(r"(?<!\^)Vea\b") : "Aa^Vea",
    re.compile(r"(?<!\^)Veg\b") : "Gg^Ve",
    re.compile(r"(?<!\^)Vha\b") : "Aa^Vha",
    re.compile(r"(?<!\^)Vhg\b") : "Gg^Vh",
    re.compile(r"(?<!\^)Vhh\b") : "Hh^Vhh",
    re.compile(r"(?<!\^)Vhha\b") : "Ha^Vhha",
    re.compile(r"(?<!\^)Vhm\b") : "Mm^Vhh",
    re.compile(r"(?<!\^)Vht\b") : "Gs^Vht",
    re.compile(r"(?<!\^)Vu\b") : "Uu^Vu",
    re.compile(r"(?<!\^)Vud\b") : "Uu^Vud",
    re.compile(r"(?<!\^)Vwm\b") : "Ww^Vm",
    re.compile(r"(?<!\^)Vs\b") : "Ss^Vhs",
    re.compile(r"(?<!\^)Vsm\b") : "Ss^Vm",
    re.compile(r"(?<!\^)Xm\b") : "Mm^Xm",
    }

def maptransform2(filename, baseline, inmap, y):
    "Convert a map line from 1.3.1 multiletter format to 1.3.2 format."
    for x in range(len(inmap[y])):
        # General conversions
        for (old, new) in conversion2.items():
            inmap[y][x] = old.sub(new, inmap[y][x])
        # Convert keeps according to adjacent hexes
        if "_K" in inmap[y][x]:
            adj = map(string.strip, neighborhood(x, y, inmap))

            # print "adjacent: %s" % adj
            hexcount = {}
            # Intentionally skipping 0 as it is original hex
            for i in range(1, len(adj)):
                if adj[i].startswith("C"): # this is a castle hex
                    # Magic: extract second character of each adjacent castle,
                    # which is its base type.  Count occurrences of each type.
                    basetype = adj[i][1]
                    hexcount[basetype] = hexcount.get(basetype, 0) + 1
            maxc = 0;
            maxk = "h";
            # Note: if two kinds of basetype tie for most instances adjacent,
            # which one dominates will be a pseudorandom artifact of
            # Python's hash function.
            for k in hexcount.keys():
                if hexcount[k] > maxc:
                    maxc = hexcount[k]
                    maxk = k
            #print "Dominated by %s" % maxk
            inmap[y][x] = inmap[y][x].replace("_K", "K" + maxk)
            # There's only one kind of underground keep at present.
            inmap[y][x] = inmap[y][x].replace("Ku", "Kud")

name_in_attack = False

# These hooks are not presently used.

def validate_stack(stack, filename, lineno):
    "Check the stack for deprecated WML syntax."
    if verbose >= 3:
        print '"%s", line %d: %s' % (filename, lineno+1, stack)
    if stack:
        (tag, attributes) = tagstack[-1]
        ancestors = map(lambda x: x[0], tagstack)
        if tag == "sound" and "attack" in ancestors:
            print '"%s", line %d: deprecated [sound] within [attack] tag' % (filename, lineno+1)

def validate_on_pop(tagstack, closer, filename, lineno):
    "Validate the stack at the time a new close tag is seen."
    (tag, attributes) = tagstack[-1]
    ancestors = map(lambda x: x[0], tagstack)
    if verbose >= 3:
        print '"%s", line %d: closing %s I see %s with %s' % (filename, lineno, closer, tag, attributes)
    # Detect a malformation that will cause the game to barf while attempting to
    # deserialize an empty unit.
    if closer == "side" and "type" not in attributes and ("no_leader" not in attributes or attributes["no_leader"] != "yes") and "multiplayer" not in ancestors:
        print '"%s", line %d: [side] without type attribute' % (filename, lineno)

# Syntax transformations

leading_ws = re.compile(r"^\s*")

def leader(s):
    "Return a copy of the leading whitespace in the argument."
    return leading_ws.match(s).group(0)

def outdent(s):
    "Outdent line by one level."
    if s.startswith(baseindent):
        return s[len(baseindent):]
    else:
        return s

def hack_syntax(filename, lines):
    modcount = 0
    # Ensure that every attack has a translatable description."
    for i in range(len(lines)):
        if "no-syntax-rewrite" in lines[i]:
            break
        elif "[attack]" in lines[i]:
            j = i;
            have_description = False
            while '[/attack]' not in lines[j]:
                if lines[j].strip().startswith("description"):
                    have_description = True
                j += 1
            if not have_description:
                j = i
                while '[/attack]' not in lines[j]:
                    fields = lines[j].strip().split('#')
                    syntactic = fields[0]
                    comment = ""
                    if len(fields) > 1:
                        comment = fields[1]
                    if syntactic.strip().startswith("name"):
                        description = syntactic.split("=")[1].strip()
                        if not description.startswith('"'):
                            description = '"' + description + '"\n'
                        # Skip the insertion if this is a dummy declaration
                        # or one modifying an attack inherited from a base unit.
                        if "no-icon" not in comment:
                            new_line = leader(syntactic) + "description=_"+description
                            if verbose:
                                print '"%s", line %d: inserting %s' % (filename, i+1, `new_line`)
                            lines.insert(j+1, new_line)
                            j += 1
                            modcount += 1
                    j += 1
    # Ensure that every speaker=narrator block without an image uses
    # wesnoth-icon.png as an image.
    need_image = False
    for i in range(len(lines)):
        if "no-syntax-rewrite" in lines[i]:
            break
        precomment = lines[i].split("#")[0]
        if "speaker=narrator" in precomment:
            need_image = True
        elif precomment.strip().startswith("image"):
            need_image = False
        elif '[/message]' in precomment:
            if need_image:
                # This line presumes the code has been through wmlindent
                if verbose:
                    print 'wmllint: "%s", line %d: inserting "image=wesnoth-icon.png"'%(filename, i+1)
                lines.insert(i, leader(precomment) + baseindent + "image=wesnoth-icon.png\n")
                modcount += 1        
            need_image = False
    # Boucman's transformation of animation syntax
    if future:
        class anim_frame:
            def __init__(self, attackline, attackname, lineno):
                self.attackstart = attackline
                self.name = attackname
                self.animstart = lineno
                self.animend = None
                self.attackend = None
            def __repr__(self):
                return `self.__dict__`
        in_attack = in_animation = in_female = False
        animations = []
        attackname = None
        attackline = None
        for i in range(len(lines)):
            if "no-syntax-rewrite" in lines[i]:
                break
            elif "[female]" in lines[i]:
                in_female = True
            elif "[/female]" in lines[i]:
                in_female = False
            elif "[unit]" in lines[i]:
                in_attack = in_animation = in_female = False
                female_attack_index = -1
                male_attack_start = len(animations)
            elif "[attack]" in lines[i]:
                in_attack = True;
                attackname = None
                attackline = i
                if in_female:
                    female_attack_index += 1
            elif "[animation]" in lines[i] and in_attack:
                #if verbose:
                #    print '"%s", line %d: [animation] within [attack]' \
                #          % (filename, i+1)
                # This weird piece of code is because attacks for female
                # variants don't have names.  Instead, they're supposed
                # to pick up the name of the corresponding male attack,
                # where correspondence is by order of declaration.  The
                # male_attack_start variable copes with the possibility
                # of multiple units per file.
                if attackname == None and in_female:
                    attackname = animations[male_attack_start + female_attack_index].name
                if not attackname:
                    print '"%s", line %d: cannot deduce attack name'%(filename, i+1)
                    
                animations.append(anim_frame(attackline, attackname, i))
                in_animation = True
            elif "[/animation]" in lines[i] and in_attack:
                in_animation = False
                if animations and animations[-1].animstart != None and animations[-1].animend == None:
                    animations[-1].animend = i
                else:
                    print '"%s", line %d: [animation] ending here may be ill-formed'%(filename, i+1)
            elif "[/attack]" in lines[i]:
                inattack = False;
                attackname = None
                if animations and (animations[-1].attackstart == None or animations[-1].attackend != None):
                    print '"%s", line %d: [attack] ending here may be ill-formed'%(filename, i+1)
                elif animations:
                    # This loop is needed because a single attack tag may
                    # enclose both hit and miss animations.
                    j = len(animations)-1
                    while True:
                        animations[j].attackend = i
                        j -= 1
                        if j < 0 or animations[j].attackend != None:
                            break
            # Only pick up the *first* name field in an attack block;
            # by convention, it will be right after the opening [attack] tag
            elif in_attack and not in_animation and not attackname:
                #print filename + ":" + `i+1` + ";" + `lines[i]` 
                fields = lines[i].strip().split('#')
                syntactic = fields[0]
                comment = ""
                if len(fields) > 1:
                    comment = fields[1]
                if syntactic.strip().startswith("name"):
                    attackname = syntactic.split("=")[1].strip()
        # All animation ranges have been gathered,  We have a list of tuples
        # of the form [attack_startline, attack_name, animation_startline,
        # animation_endline, attack_endline].  Reverse it, because we're
        # going to process them back to front to avoid invalidating the
        # already-collected line numbers.
        animations.reverse()
        attack_anims = []
        for aframe in animations:
            if verbose:
                print '"%s", line %d: lifting animation block at %d:%d for %s attack (%d:%d)' % (filename, aframe.animstart+1, aframe.animstart+1, aframe.animend+1, aframe.name, aframe.attackstart+1, aframe.attackend+1)
            # Make a copy of the animation block, change its enclosing tags,
            # outdent it, and add the needed filter clause.
            animation = lines[aframe.animstart:aframe.animend+1]
            animation[0] = animation[0].replace("[animation]", "[attack_anim]")
            animation[-1] = animation[-1].replace("[/animation]","[/attack_anim]")
            for i in range(len(animation)):
                animation[i] = outdent(animation[i])
            indent = leader(animation[1])
            animation.insert(1, indent + "[/attack_filter]\n")
            animation.insert(1, indent + baseindent + "name="+aframe.name+"\n")
            animation.insert(1, indent + "[attack_filter]\n")
            # Save it and delete it from its original location
            attack_anims = animation + attack_anims
            lines = lines[:aframe.animstart] + lines[aframe.animend+1:]
            modcount += 1
        # Add the attack animations to the end of the unit declaration
        if attack_anims:
            unitend = -1
            for i in range(len(lines)):
                if lines[i].endswith("[/unit]\n"):
                    unitend = i
            assert unitend != -1
            lines = lines[:unitend] + attack_anims + lines[unitend:]
        # Garbage-collect any empty [attack] scopes left behind;
        # this is likely to happen with female-variant units.
        nullattack = True
        while nullattack:
            nullattack = False
            for i in range(len(lines)-1):
                if lines[i].strip() == "[attack]" and lines[i+1].strip() == "[/attack]":
                    nullattack = True
            if nullattack:
                lines = lines[:i] + lines[i+2:] 
    return (lines, modcount)

# Generic machinery starts here

class maptransform_error:
    "Error object to be thrown by maptransform."
    def __init__(self, infile, inline, type):
        self.infile = infile
        self.inline = inline
        self.type = type
    def __repr__(self):
        return '"%s", line %d: %s' % (self.infile, self.inline, self.type)

tagstack = []	# For tracking tag nesting

def translator(filename, mapxforms, textxform):
    "Apply mapxform to map lines and textxform to non-map lines."
    global tagstack
    modified = False
    mfile = []
    map_only = not filename.endswith(".cfg")
    terminator = "\n"
    for line in open(filename):
        if line.endswith("\n"):
            line = line[:-1]
        if line.endswith("\r"):
            line = line[:-1]
            if not stripcr:
                terminator = '\r\n'
        mfile.append(line)
        if "map_data" in line:
            map_only = False
    cont = False
    outmap = []
    newdata = []
    lineno = baseline = 0
    validate = True
    while mfile:
        if not map_only:
            line = mfile.pop(0)
            if verbose >= 3:
                sys.stdout.write(line + terminator)
            lineno += 1
        # Exclude map_data= lines that are just 1 line without
        # continuation, or which contain {}.  The former are
        # pathological and the parse won't handle them, the latter
        # refer to map files which will be checked separately.
        if map_only or ("map_data=" in line
                        and line.count('"') in (1, 2)
                        and line.count("{") == 0
                        and  line.count("}") == 0):
            baseline = 0
            cont = True
            if verbose >= 3:
                print "*** Entering map mode."
            if not map_only:
                fields = line.split('"')
                if fields[1].strip():
                    mfile.insert(0, fields[1])
                if len(fields) == 3:
                    mfile.insert(1, '"')
            while cont and mfile:
                line = mfile.pop(0)
                if verbose >= 3:
                    sys.stdout.write(line + terminator)
                lineno += 1
                if len(line) == 0 or line[0] == '#':
                    newdata.append(line + terminator)
                    continue
                if '"' in line:
                    cont = False
                    if verbose >= 3:
                        print "*** Exiting map mode."
                    line = line.split('"')[0]
                if line:
                    if ',' in line:
                        fields = line.split(",")
                    else:
                        fields = map(lambda x: x, line)
                    outmap.append(fields)
            if not map_only: 
                newdata.append("map_data=\"" + terminator)
            original = copy.deepcopy(outmap)
            for transform in mapxforms:
                for y in range(len(outmap)):
                    transform(filename, baseline, outmap, y)
            for y in range(len(outmap)):
                newdata.append(",".join(outmap[y]) + terminator)
                if original[y] != outmap[y]:
                    modified = True
            # All lines of the map are processed, add the appropriate trailer
            if not map_only:
                newdata.append("\"" + terminator)
        elif "map_data=" in line and (line.count("{") or line.count("}")):
            newline = line
            refre = re.compile(r"\{@?([^A-Z].*)\}").search(line)
            if refre:
                mapfile = refre.group(1)
                if not mapfile.endswith(".map") and modified_maps.get(mapfile)==False:
                    newline = newline.replace(mapfile, mapfile + ".map") 
            newdata.append(newline + terminator)
            if newline != line:
                modified = True
                if verbose > 0:
                    print >>sys.stderr, 'wmllint: "%s", line %d: %s -> %s.' % (filename, lineno, line, newline)
        elif "map_data=" in line and line.count('"') > 1:
            print >>sys.stderr, 'wmllint: "%s", line %d: one-line map.' % (filename, lineno)
            newdata.append(line + terminator)
        else:
            # Handle text (non-map) lines
            newline = textxform(filename, lineno, line)
            newdata.append(newline + terminator)
            if newline != line:
                modified = True
            # Now do warnings based on the state of the tag stack
            fields = newline.split("#")
            trimmed = fields[0]
            comment = ""
            if len(fields) > 1:
                comment = fields[1]
            for instance in re.finditer(r"\[\/?\+?([a-z][a-z_]*[a-z])\]", trimmed):
                tag = instance.group(1)
                attributes = []
                closer = instance.group(0)[1] == '/'
                if not closer:
                    tagstack.append((tag, {}))
                else:
                    if len(tagstack) == 0:
                        print '"%s", line %d: closer [/%s] with tag stack empty.' % (filename, lineno+1, tag) 
                    elif tagstack[-1][0] != tag:
                        print '"%s", line %d: unbalanced [%s] closed with [/%s].' % (filename, lineno+1, tagstack[-1][0], tag)
                    else:
                        if validate:
                            validate_on_pop(tagstack, tag, filename, lineno)
                        tagstack.pop()
            if tagstack:
                for instance in re.finditer(r'([a-z][a-z_]*[a-z])\s*=(\w+|"[^"]*")', trimmed):
                    attribute = instance.group(1)
                    value = instance.group(2)
                    tagstack[-1][1][attribute] = value
                if validate:
                    validate_stack(tagstack, filename, lineno)
            if "wmllint: validate-on" in comment:
                validate = True
            if "wmllint: validate-off" in comment:
                validate = False
    # It's an error if the tag stack is nonempty at the end of any file:
    if tagstack:
        print >>sys.stderr, '"%s", line %d: tag stack nonempty (%s) at end of file.' % (filename, lineno, tagstack)
    tagstack = []
    # Track which maps are modified, we'll use this later for determining
    # which files get a .map extension.
    if "maps" in filename:
        modified_maps[filename] = modified
    # OK, now perform WML rewrites
    (newdata, hacked) = hack_syntax(filename, newdata)
    # Run everything together
    filetext = "".join(newdata)
    # WML syntax changed in 1.3.5.  The transformation cannot
    # conveniently be done line-by-line.
    transformed = re.sub(r"(if]|while])\s*\[or]([\w\W]*?)\[/or]\s*",
                         r"\1\2", filetext);
    # Return None if the transformation functions made no changes.
    if modified or hacked or transformed != filetext:
        return transformed
    else:
        return None

ignore = (".tgz", ".png", ".jpg", "-bak")

def interesting(fn):
    "Is a file interesting for conversion purposes?"
    return fn.endswith(".cfg") or fn.endswith(".map") \
        or ("maps" in fn and fn[-4:] not in ignore)

def allcfgfiles(dir):
    "Get the names of all interesting files under dir."
    datafiles = []
    if not os.path.isdir(dir):
        if interesting(dir):
            if not os.path.exists(dir):
                sys.stderr.write("wmllint: %s does not exist\n" % dir)
            else:
                datafiles.append(dir)
    else:
        for root, dirs, files in os.walk(dir):
            if vcdir in dirs:
                dirs.remove(vcdir)
            for name in files:
                if interesting(os.path.join(root, name)):
                    datafiles.append(os.path.join(root, name))
    datafiles.sort()	# So diffs for same campaigns will cluster in reports
    return map(os.path.normpath, datafiles)

def help():
        sys.stderr.write("""\
Usage: wmllint [options] [dir]
    Convert Battle of Wesnoth WML from older versions to newer ones.
    Takes any number of directories as arguments.  Each directory is converted.
    If no directories are specified, acts on the current directory.
    Options may be any of these:
    -h, --help                 Emit this help message and quit.
    -d, --dryrun               List changes but don't perform them.
    -o, --oldversion           Specify version to begin with.
    -v, --verbose              -v        lists changes.
                               -v -v     names each file before it's processed.
                               -v -v -v  shows verbose parse details.
    -c, --clean                Clean up -bak files.
    -D, --diff                 Display diffs between converted and unconverted files.
    -r, --revert               Revert the conversion from the -bak files.
    -s, --stripcr              Convert DOS-style CR/LF to Unix-style LF.
    --future                   Enable experimental WML conversions.
""")

if __name__ == '__main__':
    try:
        (options, arguments) = getopt.getopt(sys.argv[1:], "cdfDho:rsv", [
            "help",
            "oldversion=",
            "dryrun",
            "future",
            "verbose",
            "clean",
            "revert",
            "diffs",
            "stripcr",
            ])
    except getopt.GetoptError:
        help()
        sys.exit(1)
    oldversion = 'older'
    dryrun = False
    future = False
    verbose = 0
    clean = False
    diffs = False
    revert = False
    stripcr = False
    for (switch, val) in options:
        if switch in ('-h', '--help'):
            help()
            sys.exit(0)
        elif switch in ('-o', '--oldversion'):
            oldversion = val
        elif switch in ('-f', '--future'):
            future = True
        elif switch in ('-v', '--verbose'):
            verbose += 1
        elif switch in ('-d', '--dryrun'):
            dryrun = True
            verbose = max(1, verbose)
        elif switch in ('-c', '--clean'):
            clean = True
        elif switch in ('-d', '--diffs'):
            diffs = True
        elif switch in ('-r', '--revert'):
            revert = True
        elif switch in ('-s', '--stripcr'):
            stripcr = True

    if clean and revert:
        sys.stderr.write("wmllint: can't do clean and revert together.\n")
        sys.exit(1)

    # Compute the series of version upgrades to perform, and describe it.
    versions = filemoves.keys()
    versions.sort()
    versions = [versions[-1]] + versions[:-1]	# Move 'older' to front
    if oldversion in versions:
        versions = versions[versions.index(oldversion):]
    else:
        print >>sys.stderr, "wmllint: unrecognized version."
        sys.exit(1)
    if not dryrun and not clean and not revert:
        explain = "Upgrades for:"
        for i in range(len(versions)-1):
            explain += " %s -> %s," % (versions[i],  versions[i+1])
        sys.stdout.write(explain[:-1] + ".\n")
    fileconversions = map(lambda x: filemoves[x], versions[:-1])

    def hasdigit(str):
        for c in str:
            if c in "0123456789":
                return True
        return False

    def parse_attribute(str):
        "Parse a WML key-value pair from a line."
        if '=' not in str:
            return None
        m = re.match(r"(^\s*[a-z0-9_]+\s*=\s*)(\S+)(\s*#?.*\s*)", str)
        if not m:
            return None
        # Four fields: stripped key, part of line before value,
        # value, trailing whitespace and comments
        return (m.group(1).replace("=", "").strip(),) + m.groups()

    def texttransform(filename, lineno, line):
        "Resource-name transformation on text lines."
        transformed = line
        # First, do resource-file moves
        for step in fileconversions:
            for (old, new) in step:
                transformed = old.sub(new, transformed)
        # Handle terrain_liked=, terrain=, valid_terrain=, letter=
        spaceless = transformed.replace(" ", "").replace("\t", "")
        if spaceless and spaceless[0] != "#" and ("terrain_liked=" in spaceless or "terrain=" in spaceless or 'letter=' in spaceless) and "wmllint:ignore" not in spaceless:
            (key, pre, value, post) = parse_attribute(transformed)
            # We have to cope with the following cases...
            # Old style:
            #    terrain_liked=ghM
            #    terrain_liked=BEITU
            #    valid_terrain=gfh
            #    terrain=AaBbDeLptUVvYZ
            #    terrain=r
            #    terrain={LETTERS}
            #    terrain=""
            #    terrain=s,c,w,k
            # New style:
            #    terrain=Mm
            #    terrain=Gs^Fp
            #    terrain=Hh, Gg^Vh, Mm
            # The sticky part is that, while it never happens in the current
            # corpus, terrain=Mm (capital letter followed by small) could be
            # interpreted either way.
            #
            # There are some unambiguous tests: 
            oldstyle = (len(value) == 1 or len(value) > 6) and not ',' in value
            newstyle = len(value) > 1 \
                       and value[0].isupper() and value[1].islower() \
                       and (',' in value \
                            or len(value) == 2 \
                            or (len(value) >= 3 and value[2] == "^"))
            # See maptransform1() for explanation of this ugly hack.
            oldstyle = oldstyle or lock_terrain_coding == "oldstyle"
            newstyle = newstyle or lock_terrain_coding == "newstyle"
            # Maybe we lose...
            if not oldstyle and not newstyle:
                print '"%s", line %d: leaving ambiguous terrain value %s alone.' \
                      % (filename, lineno+1, value)
            else:
                if oldstyle:
                    # 1.2.x to 1.3.2 conversions
                    newterrains = ""
                    inmacro = False
                    for c in value:
                        if not inmacro:
                            if c == '{':
                                inmacro = True
                                newterrains += c                            
                            elif c == ',':
                                pass
                            elif c.isspace():
                                newterrains += c
                            elif c in conversion1:
                                newterrains += conversion1[c] + ","
                            else:
                                print "%s, line %d: custom terrain %s ignored." \
                                  % (filename, lineno+1, c)
                        else: # inmacro == True
                            if c == '}':
                                inmacro = False
                            newterrains += c
                    if newterrains.endswith(","):
                        newterrains = newterrains[:-1]
                    transformed = pre + newterrains + post
                if newstyle:
                    if len(value) == 2:
                        # 1.3.1 to 1.3.2 conversion
                        for (old, new) in conversion2.items():
                            transformed = old.sub(new, transformed)
        # Report the changes
        if verbose > 0 and transformed != line:
            msg = "%s, line %d: %s -> %s" % \
                  (filename, lineno+1, line.strip(), transformed.strip())
            print msg
        return transformed

    if "1.3.1" in versions and "older" not in versions:
        maptransforms = [maptransform2]
    else:
        maptransforms = [maptransform1, maptransform2]

    if not arguments:
        arguments = ["."]

    for dir in arguments:
        ofp = None
        modified_maps = {}
        if "older" in versions:
            lock_terrain_coding = None
        else:
            lock_terrain_coding = "newstyle"
        for fn in allcfgfiles(dir):
            if verbose >= 2:
                print fn + ":"
            backup = fn + "-bak"
            if clean or revert:
                # Do housekeeping
                if os.path.exists(backup):
                    if clean:
                        print "wmllint: removing %s" % backup
                        if not dryrun:
                            os.remove(backup)
                    elif revert:
                        print "wmllint: reverting %s" % backup
                        if not dryrun:
                            os.rename(backup, fn)
            elif diffs:
                # Display diffs
                if os.path.exists(backup):
                    fromdate = time.ctime(os.stat(backup).st_mtime)
                    todate = time.ctime(os.stat(fn).st_mtime)
                    fromlines = open(backup, 'U').readlines()
                    tolines = open(fn, 'U').readlines()
                    diff = difflib.unified_diff(fromlines, tolines,
                                         backup, fn, fromdate, todate, n=3)
                    sys.stdout.writelines(diff)
            else:
                # Do file conversions
                try:
                    changed = translator(fn, maptransforms, texttransform)
                    if changed:
                        print "wmllint: converting", fn
                        if not dryrun:
                            os.rename(fn, backup)
                            ofp = open(fn, "w")
                            ofp.write(changed)
                            ofp.close()
                except maptransform_error, e:
                    sys.stderr.write("wmllint: " + `e` + "\n")
                except:
                    sys.stderr.write("wmllint: internal error on %s\n" % fn)
                    (exc_type, exc_value, exc_traceback) = sys.exc_info()
                    raise exc_type, exc_value, exc_traceback
            # Time for map file renames
            if not fn.endswith(".map") and modified_maps.get(fn) == False:
                mover = vcmove(fn, fn + ".map")
                print mover
                if not dryrun:
                    os.system(mover)

# wmllint ends here
