Files

 master

/

src

/

Copy path

Directory actions

More options

Directory actions

More options

Latest commit

vrnmthr

update arch

Jul 15, 2024

657cab5 · Jul 15, 2024

History

History

This branch is 22 commits ahead of, 362 commits behind google/sentencepiece:master.

/

src

/

Top

Folders and files

Name		Name	Last commit message	Last commit date
parent directory ..
builtin_pb		builtin_pb	Enables --encode_unicode_case option for case-aware sentence piece (#10)	Jun 16, 2021
CMakeLists.txt		CMakeLists.txt	link android logger lib	May 27, 2024
bpe_model.cc		bpe_model.cc	validate the range of piece in Python module	Oct 23, 2020
bpe_model.h		bpe_model.h	clear description for alpha of BPE-dropout	Sep 4, 2020
bpe_model_test.cc		bpe_model_test.cc	Fixed windows build failure	May 9, 2020
bpe_model_trainer.cc		bpe_model_trainer.cc	fix typo	Aug 27, 2020
bpe_model_trainer.h		bpe_model_trainer.h	Port absl::flat_hash_map	Jun 1, 2020
bpe_model_trainer_test.cc		bpe_model_trainer_test.cc	Use absl::flags	May 31, 2020
builder.cc		builder.cc	Enables --encode_unicode_case option for case-aware sentence piece (#10)	Jun 16, 2021
builder.h		builder.h	Enables --encode_unicode_case option for case-aware sentence piece (#10)	Jun 16, 2021
builder_test.cc		builder_test.cc	Use absl::flags	May 31, 2020
case_encoder.cc		case_encoder.cc	Enables --encode_unicode_case option for case-aware sentence piece (#10)	Jun 16, 2021
case_encoder.h		case_encoder.h	Fix Surface String to Token Mappings for Case Encoding (#12)	Aug 31, 2021
char_model.cc		char_model.cc	stop normalization for user_defined_symbols	Nov 8, 2018
char_model.h		char_model.h	Port absl::flat_hash_map	Jun 1, 2020
char_model_test.cc		char_model_test.cc	Initial release of 0.19. Merged internal sentencepiece.	May 7, 2020
char_model_trainer.cc		char_model_trainer.cc	merges internal changes to github	Oct 13, 2020
char_model_trainer.h		char_model_trainer.h	Port absl::flat_hash_map	Jun 1, 2020
char_model_trainer_test.cc		char_model_trainer_test.cc	Use absl::flags	May 31, 2020
common.h		common.h	Port absl::flat_hash_map	Jun 1, 2020
compile_charsmap_main.cc		compile_charsmap_main.cc	Enables --encode_unicode_case option for case-aware sentence piece (#10)	Jun 16, 2021
error.cc		error.cc	Initial release of 0.19. Merged internal sentencepiece.	May 7, 2020
filesystem.cc		filesystem.cc	Initial release of 0.19. Merged internal sentencepiece.	May 7, 2020
filesystem.h		filesystem.h	Initial release of 0.19. Merged internal sentencepiece.	May 7, 2020
filesystem_test.cc		filesystem_test.cc	Use absl::flags	May 31, 2020
freelist.h		freelist.h	Surpress build warning, reproduced minloglevel	May 23, 2020
freelist_test.cc		freelist_test.cc	Port absl::flat_hash_map	Jun 1, 2020
init.cc		init.cc	Use absl::flags	May 31, 2020
init.h		init.h	Use absl::flags	May 31, 2020
init_test.cc		init_test.cc	Port absl::flat_hash_map	Jun 1, 2020
model_factory.cc		model_factory.cc	Initial release of 0.19. Merged internal sentencepiece.	May 7, 2020
model_factory.h		model_factory.h	Port absl::flat_hash_map	Jun 1, 2020
model_factory_test.cc		model_factory_test.cc	Initialize repository	Mar 7, 2017
model_interface.cc		model_interface.cc	Port absl::flat_hash_map	Jun 1, 2020
model_interface.h		model_interface.h	Port absl::flat_hash_map	Jun 1, 2020
model_interface_test.cc		model_interface_test.cc	Port absl::flat_hash_map	Jun 1, 2020
normalization_rule.h		normalization_rule.h	Enables --encode_unicode_case option for case-aware sentence piece (#10)	Jun 16, 2021
normalizer.cc		normalizer.cc	Enables --encode_unicode_case option for case-aware sentence piece (#10)	Jun 16, 2021
normalizer.h		normalizer.h	Enables --encode_unicode_case option for case-aware sentence piece (#10)	Jun 16, 2021
normalizer_test.cc		normalizer_test.cc	support big-endian architecture	Oct 13, 2020
pretokenizer_for_training.cc		pretokenizer_for_training.cc	Initial release of 0.19. Merged internal sentencepiece.	May 7, 2020
pretokenizer_for_training.h		pretokenizer_for_training.h	Port absl::flat_hash_map	Jun 1, 2020
pretokenizer_for_training_test.cc		pretokenizer_for_training_test.cc	Initial release of 0.19. Merged internal sentencepiece.	May 7, 2020
sentencepiece.proto		sentencepiece.proto	Initial release of 0.19. Merged internal sentencepiece.	May 7, 2020
sentencepiece_model.proto		sentencepiece_model.proto	Enables --encode_unicode_case option for case-aware sentence piece (#10)	Jun 16, 2021
sentencepiece_processor.cc		sentencepiece_processor.cc	Fix Surface String to Token Mappings for Case Encoding (#12)	Aug 31, 2021
sentencepiece_processor.h		sentencepiece_processor.h	validate the range of piece in Python module	Oct 23, 2020
sentencepiece_processor_test.cc		sentencepiece_processor_test.cc	merges internal changes to github	Oct 13, 2020
sentencepiece_trainer.cc		sentencepiece_trainer.cc	Disable denormalizer flags (#13)	Aug 27, 2021
sentencepiece_trainer.h		sentencepiece_trainer.h	Fix build break.	Jun 2, 2020
sentencepiece_trainer_test.cc		sentencepiece_trainer_test.cc	Port absl::flat_hash_map	Jun 1, 2020
spec_parser.h		spec_parser.h	Enables --encode_unicode_case option for case-aware sentence piece (#10)	Jun 16, 2021
spm_decode_main.cc		spm_decode_main.cc	Port absl::flat_hash_map	Jun 1, 2020
spm_encode_main.cc		spm_encode_main.cc	add SetRandomGeneratorSeed	Oct 24, 2020
spm_export_vocab_main.cc		spm_export_vocab_main.cc	Port absl::flat_hash_map	Jun 1, 2020
spm_normalize_main.cc		spm_normalize_main.cc	Port absl::flat_hash_map	Jun 1, 2020
spm_train_main.cc		spm_train_main.cc	Enables --encode_unicode_case option for case-aware sentence piece (#10)	Jun 16, 2021
test_main.cc		test_main.cc	Use absl::flags	May 31, 2020
testharness.cc		testharness.cc	Use absl::flags	May 31, 2020
testharness.h		testharness.h	Use absl::flags	May 31, 2020
trainer_factory.cc		trainer_factory.cc	Initial release of 0.19. Merged internal sentencepiece.	May 7, 2020
trainer_factory.h		trainer_factory.h	Port absl::flat_hash_map	Jun 1, 2020
trainer_factory_test.cc		trainer_factory_test.cc	Initial release of 0.19. Merged internal sentencepiece.	May 7, 2020
trainer_interface.cc		trainer_interface.cc	Enables --encode_unicode_case option for case-aware sentence piece (#10)	Jun 16, 2021
trainer_interface.h		trainer_interface.h	Port absl::flat_hash_map	Jun 1, 2020
trainer_interface_test.cc		trainer_interface_test.cc	Port absl::flat_hash_map	Jun 1, 2020
unicode_script.cc		unicode_script.cc	Port absl::flat_hash_map	Jun 1, 2020
unicode_script.h		unicode_script.h	Initialize repository	Mar 7, 2017
unicode_script_map.h		unicode_script_map.h	Port absl::flat_hash_map	Jun 1, 2020
unicode_script_test.cc		unicode_script_test.cc	Initial release of 0.19. Merged internal sentencepiece.	May 7, 2020
unigram_model.cc		unigram_model.cc	0.1.91 pre-release	May 20, 2020
unigram_model.h		unigram_model.h	Port absl::flat_hash_map	Jun 1, 2020
unigram_model_test.cc		unigram_model_test.cc	Port absl::flat_hash_map	Jun 1, 2020
unigram_model_trainer.cc		unigram_model_trainer.cc	merges internal changes to github	Oct 13, 2020
unigram_model_trainer.h		unigram_model_trainer.h	Port absl::flat_hash_map	Jun 1, 2020
unigram_model_trainer_test.cc		unigram_model_trainer_test.cc	Port absl::flat_hash_map	Jun 1, 2020
util.cc		util.cc	add SetRandomGeneratorSeed	Oct 24, 2020
util.h		util.h	update arch	Jul 15, 2024
util_test.cc		util_test.cc	Use absl::flags	May 31, 2020
word_model.cc		word_model.cc	Initial release of 0.19. Merged internal sentencepiece.	May 7, 2020
word_model.h		word_model.h	Port absl::flat_hash_map	Jun 1, 2020
word_model_test.cc		word_model_test.cc	Port absl::flat_hash_map	Jun 1, 2020
word_model_trainer.cc		word_model_trainer.cc	merges internal changes to github	Oct 13, 2020
word_model_trainer.h		word_model_trainer.h	Port absl::flat_hash_map	Jun 1, 2020
word_model_trainer_test.cc		word_model_trainer_test.cc	Use absl::flags	May 31, 2020
View all files