1

/* $Id$ */

     2

     3

/** @file strgen.cpp Tool to create computer readable (stand-alone) translation files. */

     4

     5

#include "../stdafx.h"

     6

#include "../core/alloc_func.hpp"

     7

#include "../core/endian_func.hpp"

     8

#include "../string_func.h"

     9

#include "../strings_type.h"

    10

#include "strgen.h"

    11

#include "../table/control_codes.h"

    12

    13

#include <stdio.h>

    14

#include <string.h>

    15

#include <stdlib.h>

    16

#include <stdarg.h>

    17

    18

#if (!defined(WIN32) && !defined(WIN64)) || defined(__CYGWIN__)

    19

#include <unistd.h>

    20

#include <sys/types.h>

    21

#include <sys/stat.h>

    22

#endif

    23

    24

#if defined WIN32 || defined __WATCOMC__

    25

#include <direct.h>

    26

#endif /* WIN32 || __WATCOMC__ */

    27

    28

#ifdef __MORPHOS__

    29

#ifdef stderr

    30

#undef stderr

    31

#endif

    32

#define stderr stdout

    33

#endif /* __MORPHOS__ */

    34

    35

/* Compiles a list of strings into a compiled string list */

    36

    37

typedef void (*ParseCmdProc)(char *buf, int value);

    38

    39

struct CmdStruct {

    40

	const char *cmd;

    41

	ParseCmdProc proc;

    42

	long value;

    43

	int8 consumes;

    44

	byte flags;

    45

};

    46

    47

enum {

    48

	C_DONTCOUNT = 1,

    49

	C_CASE      = 2,

    50

};

    51

    52

    53

struct Case {

    54

	int caseidx;

    55

	char *string;

    56

	Case *next;

    57

};

    58

    59

static bool _masterlang;

    60

static bool _translated;

    61

static const char* _file = "(unknown file)";

    62

static int _cur_line;

    63

static int _errors, _warnings, _show_todo;

    64

    65

struct LangString {

    66

	char *name;            // Name of the string

    67

	char *english;         // English text

    68

	char *translated;      // Translated text

    69

	uint16 hash_next;      // next hash entry

    70

	uint16 index;

    71

	int line;              // line of string in source-file

    72

	Case *english_case;    // cases for english

    73

	Case *translated_case; // cases for foreign

    74

};

    75

    76

static LangString *_strings[65536];

    77

    78

    79

#define HASH_SIZE 32767

    80

static uint16 _hash_head[HASH_SIZE];

    81

    82

static byte _put_buf[4096];

    83

static int _put_pos;

    84

static int _next_string_id;

    85

    86

static uint32 _hash;

    87

static char _lang_name[32], _lang_ownname[32], _lang_isocode[16];

    88

static byte _lang_pluralform;

    89

static byte _lang_textdir;

    90

static uint16 _lang_winlangid;

    91

#define MAX_NUM_GENDER 8

    92

static char _genders[MAX_NUM_GENDER][16];

    93

static int _numgenders;

    94

    95

// contains the name of all cases.

    96

#define MAX_NUM_CASES 50

    97

static char _cases[MAX_NUM_CASES][16];

    98

static int _numcases;

    99

   100

// for each plural value, this is the number of plural forms.

   101

static const byte _plural_form_counts[] = { 2, 1, 2, 3, 3, 3, 3, 3, 4, 2 };

   102

   103

static const char *_cur_ident;

   104

   105

struct CmdPair {

   106

	const CmdStruct *a;

   107

	const char *v;

   108

};

   109

   110

struct ParsedCommandStruct {

   111

	int np;

   112

	CmdPair pairs[32];

   113

	const CmdStruct *cmd[32]; // ordered by param #

   114

};

   115

   116

// Used when generating some advanced commands.

   117

static ParsedCommandStruct _cur_pcs;

   118

static int _cur_argidx;

   119

   120

static uint HashStr(const char *s)

   121

{

   122

	uint hash = 0;

   123

	for (; *s != '\0'; s++) hash = ROL(hash, 3) ^ *s;

   124

	return hash % HASH_SIZE;

   125

}

   126

   127

static void HashAdd(const char *s, LangString *ls)

   128

{

   129

	uint hash = HashStr(s);

   130

	ls->hash_next = _hash_head[hash];

   131

	_hash_head[hash] = ls->index + 1;

   132

}

   133

   134

static LangString *HashFind(const char *s)

   135

{

   136

	int idx = _hash_head[HashStr(s)];

   137

   138

	while (--idx >= 0) {

   139

		LangString* ls = _strings[idx];

   140

   141

		if (strcmp(ls->name, s) == 0) return ls;

   142

		idx = ls->hash_next;

   143

	}

   144

	return NULL;

   145

}

   146

   147

#ifdef _MSC_VER

   148

# define LINE_NUM_FMT "(%d)"

   149

#else

   150

# define LINE_NUM_FMT ":%d"

   151

#endif

   152

   153

static void CDECL strgen_warning(const char *s, ...)

   154

{

   155

	char buf[1024];

   156

	va_list va;

   157

	va_start(va, s);

   158

	vsnprintf(buf, lengthof(buf), s, va);

   159

	va_end(va);

   160

	fprintf(stderr, "%s" LINE_NUM_FMT ": warning: %s\n", _file, _cur_line, buf);

   161

	_warnings++;

   162

}

   163

   164

static void CDECL strgen_error(const char *s, ...)

   165

{

   166

	char buf[1024];

   167

	va_list va;

   168

	va_start(va, s);

   169

	vsnprintf(buf, lengthof(buf), s, va);

   170

	va_end(va);

   171

	fprintf(stderr, "%s" LINE_NUM_FMT ": error: %s\n", _file, _cur_line, buf);

   172

	_errors++;

   173

}

   174

   175

void NORETURN CDECL error(const char *s, ...)

   176

{

   177

	char buf[1024];

   178

	va_list va;

   179

	va_start(va, s);

   180

	vsnprintf(buf, lengthof(buf), s, va);

   181

	va_end(va);

   182

	fprintf(stderr, "%s" LINE_NUM_FMT ": FATAL: %s\n", _file, _cur_line, buf);

   183

	exit(1);

   184

}

   185

   186

static void PutByte(byte c)

   187

{

   188

	if (_put_pos == lengthof(_put_buf)) error("Put buffer too small");

   189

	_put_buf[_put_pos++] = c;

   190

}

   191

   192

   193

static void PutUtf8(uint32 value)

   194

{

   195

	if (value < 0x80) {

   196

		PutByte(value);

   197

	} else if (value < 0x800) {

   198

		PutByte(0xC0 + GB(value,  6, 5));

   199

		PutByte(0x80 + GB(value,  0, 6));

   200

	} else if (value < 0x10000) {

   201

		PutByte(0xE0 + GB(value, 12, 4));

   202

		PutByte(0x80 + GB(value,  6, 6));

   203

		PutByte(0x80 + GB(value,  0, 6));

   204

	} else if (value < 0x110000) {

   205

		PutByte(0xF0 + GB(value, 18, 3));

   206

		PutByte(0x80 + GB(value, 12, 6));

   207

		PutByte(0x80 + GB(value,  6, 6));

   208

		PutByte(0x80 + GB(value,  0, 6));

   209

	} else {

   210

		strgen_warning("Invalid unicode value U+0x%X", value);

   211

	}

   212

}

   213

   214

   215

size_t Utf8Validate(const char *s)

   216

{

   217

	uint32 c;

   218

   219

	if (!HasBit(s[0], 7)) {

   220

		/* 1 byte */

   221

		return 1;

   222

	} else if (GB(s[0], 5, 3) == 6 && IsUtf8Part(s[1])) {

   223

		/* 2 bytes */

   224

		c = GB(s[0], 0, 5) << 6 | GB(s[1], 0, 6);

   225

		if (c >= 0x80) return 2;

   226

	} else if (GB(s[0], 4, 4) == 14 && IsUtf8Part(s[1]) && IsUtf8Part(s[2])) {

   227

		/* 3 bytes */

   228

		c = GB(s[0], 0, 4) << 12 | GB(s[1], 0, 6) << 6 | GB(s[2], 0, 6);

   229

		if (c >= 0x800) return 3;

   230

	} else if (GB(s[0], 3, 5) == 30 && IsUtf8Part(s[1]) && IsUtf8Part(s[2]) && IsUtf8Part(s[3])) {

   231

		/* 4 bytes */

   232

		c = GB(s[0], 0, 3) << 18 | GB(s[1], 0, 6) << 12 | GB(s[2], 0, 6) << 6 | GB(s[3], 0, 6);

   233

		if (c >= 0x10000 && c <= 0x10FFFF) return 4;

   234

	}

   235

   236

	return 0;

   237

}

   238

   239

   240

static void EmitSingleChar(char *buf, int value)

   241

{

   242

	if (*buf != '\0') strgen_warning("Ignoring trailing letters in command");

   243

	PutUtf8(value);

   244

}

   245

   246

   247

static void EmitSetX(char *buf, int value)

   248

{

   249

	char *err;

   250

	int x = strtol(buf, &err, 0);

   251

	if (*err != 0) error("SetX param invalid");

   252

	PutUtf8(SCC_SETX);

   253

	PutByte((byte)x);

   254

}

   255

   256

   257

static void EmitSetXY(char *buf, int value)

   258

{

   259

	char *err;

   260

	int x;

   261

	int y;

   262

   263

	x = strtol(buf, &err, 0);

   264

	if (*err != ' ') error("SetXY param invalid");

   265

	y = strtol(err + 1, &err, 0);

   266

	if (*err != 0) error("SetXY param invalid");

   267

   268

	PutUtf8(SCC_SETXY);

   269

	PutByte((byte)x);

   270

	PutByte((byte)y);

   271

}

   272

   273

// The plural specifier looks like

   274

// {NUM} {PLURAL -1 passenger passengers} then it picks either passenger/passengers depending on the count in NUM

   275

   276

// This is encoded like

   277

//  CommandByte <ARG#> <NUM> {Length of each string} {each string}

   278

   279

bool ParseRelNum(char **buf, int *value)

   280

{

   281

	const char* s = *buf;

   282

	char* end;

   283

	bool rel = false;

   284

	int v;

   285

   286

	while (*s == ' ' || *s == '\t') s++;

   287

	if (*s == '+') {

   288

		rel = true;

   289

		s++;

   290

	}

   291

	v = strtol(s, &end, 0);

   292

	if (end == s) return false;

   293

	if (rel || v < 0) {

   294

		*value += v;

   295

	} else {

   296

		*value = v;

   297

	}

   298

	*buf = end;

   299

	return true;

   300

}

   301

   302

// Parse out the next word, or NULL

   303

char *ParseWord(char **buf)

   304

{

   305

	char *s = *buf, *r;

   306

   307

	while (*s == ' ' || *s == '\t') s++;

   308

	if (*s == '\0') return NULL;

   309

   310

	if (*s == '"') {

   311

		r = ++s;

   312

		// parse until next " or NUL

   313

		for (;;) {

   314

			if (*s == '\0') break;

   315

			if (*s == '"') {

   316

				*s++ = '\0';

   317

				break;

   318

			}

   319

			s++;

   320

		}

   321

	} else {

   322

		// proceed until whitespace or NUL

   323

		r = s;

   324

		for (;;) {

   325

			if (*s == '\0') break;

   326

			if (*s == ' ' || *s == '\t') {

   327

				*s++ = '\0';

   328

				break;

   329

			}

   330

			s++;

   331

		}

   332

	}

   333

	*buf = s;

   334

	return r;

   335

}

   336

   337

// Forward declaration

   338

static int TranslateArgumentIdx(int arg);

   339

   340

static void EmitWordList(const char* const* words, uint nw)

   341

{

   342

	uint i;

   343

	uint j;

   344

   345

	PutByte(nw);

   346

	for (i = 0; i < nw; i++) PutByte(strlen(words[i]));

   347

	for (i = 0; i < nw; i++) {

   348

		for (j = 0; words[i][j] != '\0'; j++) PutByte(words[i][j]);

   349

	}

   350

}

   351

   352

static void EmitPlural(char *buf, int value)

   353

{

   354

	int argidx = _cur_argidx;

   355

	const char* words[5];

   356

	int nw = 0;

   357

   358

	// Parse out the number, if one exists. Otherwise default to prev arg.

   359

	if (!ParseRelNum(&buf, &argidx)) argidx--;

   360

   361

	// Parse each string

   362

	for (nw = 0; nw < 5; nw++) {

   363

		words[nw] = ParseWord(&buf);

   364

		if (words[nw] == NULL) break;

   365

	}

   366

   367

	if (nw == 0)

   368

		error("%s: No plural words", _cur_ident);

   369

   370

	if (_plural_form_counts[_lang_pluralform] != nw) {

   371

		if (_translated) {

   372

			error("%s: Invalid number of plural forms. Expecting %d, found %d.", _cur_ident,

   373

				_plural_form_counts[_lang_pluralform], nw);

   374

		} else {

   375

			if ((_show_todo & 2) != 0) strgen_warning("'%s' is untranslated. Tweaking english string to allow compilation for plural forms", _cur_ident);

   376

			if (nw > _plural_form_counts[_lang_pluralform]) {

   377

				nw = _plural_form_counts[_lang_pluralform];

   378

			} else {

   379

				for (; nw < _plural_form_counts[_lang_pluralform]; nw++) {

   380

					words[nw] = words[nw - 1];

   381

				}

   382

			}

   383

		}

   384

	}

   385

   386

	PutUtf8(SCC_PLURAL_LIST);

   387

	PutByte(TranslateArgumentIdx(argidx));

   388

	EmitWordList(words, nw);

   389

}

   390

   391

   392

static void EmitGender(char *buf, int value)

   393

{

   394

	int argidx = _cur_argidx;

   395

	uint nw;

   396

   397

	if (buf[0] == '=') {

   398

		buf++;

   399

   400

		// This is a {G=DER} command

   401

		for (nw = 0; ; nw++) {

   402

			if (nw >= 8) error("G argument '%s' invalid", buf);

   403

			if (strcmp(buf, _genders[nw]) == 0) break;

   404

		}

   405

		// now nw contains the gender index

   406

		PutUtf8(SCC_GENDER_INDEX);

   407

		PutByte(nw);

   408

	} else {

   409

		const char* words[8];

   410

   411

		// This is a {G 0 foo bar two} command.

   412

		// If no relative number exists, default to +0

   413

		if (!ParseRelNum(&buf, &argidx)) {}

   414

   415

		for (nw = 0; nw < 8; nw++) {

   416

			words[nw] = ParseWord(&buf);

   417

			if (words[nw] == NULL) break;

   418

		}

   419

		if (nw != _numgenders) error("Bad # of arguments for gender command");

   420

		PutUtf8(SCC_GENDER_LIST);

   421

		PutByte(TranslateArgumentIdx(argidx));

   422

		EmitWordList(words, nw);

   423

	}

   424

}

   425

   426

   427

static const CmdStruct _cmd_structs[] = {

   428

	// Update position

   429

	{"SETX",  EmitSetX,  SCC_SETX,  0, 0},

   430

	{"SETXY", EmitSetXY, SCC_SETXY, 0, 0},

   431

   432

	// Font size

   433

	{"TINYFONT", EmitSingleChar, SCC_TINYFONT, 0, 0},

   434

	{"BIGFONT",  EmitSingleChar, SCC_BIGFONT,  0, 0},

   435

   436

	// Colors

   437

	{"BLUE",    EmitSingleChar, SCC_BLUE,    0, 0},

   438

	{"SILVER",  EmitSingleChar, SCC_SILVER,  0, 0},

   439

	{"GOLD",    EmitSingleChar, SCC_GOLD,    0, 0},

   440

	{"RED",     EmitSingleChar, SCC_RED,     0, 0},

   441

	{"PURPLE",  EmitSingleChar, SCC_PURPLE,  0, 0},

   442

	{"LTBROWN", EmitSingleChar, SCC_LTBROWN, 0, 0},

   443

	{"ORANGE",  EmitSingleChar, SCC_ORANGE,  0, 0},

   444

	{"GREEN",   EmitSingleChar, SCC_GREEN,   0, 0},

   445

	{"YELLOW",  EmitSingleChar, SCC_YELLOW,  0, 0},

   446

	{"DKGREEN", EmitSingleChar, SCC_DKGREEN, 0, 0},

   447

	{"CREAM",   EmitSingleChar, SCC_CREAM,   0, 0},

   448

	{"BROWN",   EmitSingleChar, SCC_BROWN,   0, 0},

   449

	{"WHITE",   EmitSingleChar, SCC_WHITE,   0, 0},

   450

	{"LTBLUE",  EmitSingleChar, SCC_LTBLUE,  0, 0},

   451

	{"GRAY",    EmitSingleChar, SCC_GRAY,    0, 0},

   452

	{"DKBLUE",  EmitSingleChar, SCC_DKBLUE,  0, 0},

   453

	{"BLACK",   EmitSingleChar, SCC_BLACK,   0, 0},

   454

   455

	{"CURRCOMPACT",   EmitSingleChar, SCC_CURRENCY_COMPACT,    1, 0}, // compact currency

   456

	{"REV",           EmitSingleChar, SCC_REVISION,            0, 0}, // openttd revision string

   457

	{"SHORTCARGO",    EmitSingleChar, SCC_CARGO_SHORT,         2, 0}, // short cargo description, only ### tons, or ### litres

   458

   459

	{"STRING1", EmitSingleChar, SCC_STRING1, 2, C_CASE}, // included string that consumes the string id and ONE argument

   460

	{"STRING2", EmitSingleChar, SCC_STRING2, 3, C_CASE}, // included string that consumes the string id and TWO arguments

   461

	{"STRING3", EmitSingleChar, SCC_STRING3, 4, C_CASE}, // included string that consumes the string id and THREE arguments

   462

	{"STRING4", EmitSingleChar, SCC_STRING4, 5, C_CASE}, // included string that consumes the string id and FOUR arguments

   463

	{"STRING5", EmitSingleChar, SCC_STRING5, 6, C_CASE}, // included string that consumes the string id and FIVE arguments

   464

   465

	{"STATIONFEATURES", EmitSingleChar, SCC_STATION_FEATURES, 1, 0}, // station features string, icons of the features

   466

	{"INDUSTRY",        EmitSingleChar, SCC_INDUSTRY_NAME,    1, 0}, // industry, takes an industry #

   467

	{"CARGO",           EmitSingleChar, SCC_CARGO,            2, 0},

   468

	{"POWER",           EmitSingleChar, SCC_POWER,            1, 0},

   469

	{"VOLUME",          EmitSingleChar, SCC_VOLUME,           1, 0},

   470

	{"VOLUME_S",        EmitSingleChar, SCC_VOLUME_SHORT,     1, 0},

   471

	{"WEIGHT",          EmitSingleChar, SCC_WEIGHT,           1, 0},

   472

	{"WEIGHT_S",        EmitSingleChar, SCC_WEIGHT_SHORT,     1, 0},

   473

	{"FORCE",           EmitSingleChar, SCC_FORCE,            1, 0},

   474

	{"VELOCITY",        EmitSingleChar, SCC_VELOCITY,         1, 0},

   475

   476

	{"P", EmitPlural, 0, 0, C_DONTCOUNT}, // plural specifier

   477

	{"G", EmitGender, 0, 0, C_DONTCOUNT}, // gender specifier

   478

   479

	{"DATE_TINY",  EmitSingleChar, SCC_DATE_TINY, 1, 0},

   480

	{"DATE_SHORT", EmitSingleChar, SCC_DATE_SHORT, 1, 0},

   481

	{"DATE_LONG",  EmitSingleChar, SCC_DATE_LONG, 1, 0},

   482

   483

	{"SKIP", EmitSingleChar, SCC_SKIP, 1, 0},

   484

   485