ChangeLog

+2021-02-05  16:31  Christos Zoulas <christos@zoulas.com>
+
+	* PR/234: Add limit to the number of bytes to scan for encoding
+
 2021-02-01  12:31  Christos Zoulas <christos@zoulas.com>
 	* PR/77: Trim trailing separator.
 

doc/file.man

-.\" $File: file.man,v 1.142 2021/01/03 20:51:48 christos Exp $
-.Dd January 3, 2021
+.\" $File: file.man,v 1.143 2021/02/05 21:33:49 christos Exp $
+.Dd February 5, 2021
 .Dt FILE __CSECTION__
 .Os
 .Sh NAME
@@ -343,6 +343,7 @@ Set various parameter limits.
 .It Li elf_notes Ta 256 Ta max ELF notes processed
 .It Li elf_phnum Ta 2048 Ta max ELF program sections processed
 .It Li elf_shnum Ta 32768 Ta max ELF sections processed
+.It Li encoding Ta 65536 Ta max number of bytes to scan for encoding evaluation
 .It Li indir Ta 50 Ta recursion limit for indirect magic
 .It Li name Ta 50 Ta use count limit for name/use magic
 .It Li regex Ta 8192 Ta length limit for regex searches

magic/Magdir/bm

+
+#------------------------------------------------------------------------------
+# $File: bm,v 1.1 2021/02/05 21:54:04 christos Exp $
+# bm:  file(1) magic for "Birtual Machine", cf. https://github.com/tsoding/bm
+
+# marker is 'bm'
+0	string	bm	Birtual Machine
+>2	leshort	x	\b, version %d
+>4	lelong	x	\b, program size %u
+>12	lelong	x	\b, memory size %u
+>20	lelong	x	\b, memory capacity %u

magic/Magdir/console

 
 #------------------------------------------------------------------------------
-# $File: console,v 1.61 2020/08/09 16:38:35 christos Exp $
+# $File: console,v 1.62 2021/02/05 21:51:06 christos Exp $
 # Console game magic
 # Toby Deshane <hac@shoelace.digivill.net>
 
@@ -181,7 +181,7 @@
 # RAM size.
 >0x149 byte 1     \b, RAM: 16Kbit
 >0x149 byte 2     \b, RAM: 64Kbit
->0x149 byte 3     \b, RAM: 128Kbit
+>0x149 byte 3     \b, RAM: 256Kbit
 >0x149 byte 4     \b, RAM: 1Mbit
 >0x149 byte 5     \b, RAM: 512Kbit
 

magic/Magdir/linux

 
 #------------------------------------------------------------------------------
-# $File: linux,v 1.75 2021/01/03 20:53:52 christos Exp $
+# $File: linux,v 1.76 2021/02/05 21:58:29 christos Exp $
 # linux:  file(1) magic for Linux files
 #
 # Values for Linux/i386 binaries, from Daniel Quinlan <quinlan@yggdrasil.com>
@@ -438,17 +438,16 @@
 # Documentation/devicetree/booting-without-of.txt
 # From Christoph Biedl
 0		belong		0xd00dfeed
-# structure and strings must be within blob
+# structure must be within blob, strings are omitted to handle devicetrees > 1M
 >&(8.L)		byte		x
->>&(12.L)	byte		x
->>>20		belong		>1	Device Tree Blob version %d
->>>>4		belong		x	\b, size=%d
->>>>20		belong		>1
->>>>>28		belong		x	\b, boot CPU=%d
->>>>20		belong		>2
->>>>>32		belong		x	\b, string block size=%d
->>>>20		belong		>16
->>>>>36		belong		x	\b, DT structure block size=%d
+>>20		belong		>1	Device Tree Blob version %d
+>>>4		belong		x	\b, size=%d
+>>>20		belong		>1
+>>>>28		belong		x	\b, boot CPU=%d
+>>>20		belong		>2
+>>>>32		belong		x	\b, string block size=%d
+>>>20		belong		>16
+>>>>36		belong		x	\b, DT structure block size=%d
 
 # glibc locale archive as defined in glibc locale/locarchive.h
 0		lelong		0xde020109	locale archive

magic/Makefile.am

 #
-# $File: Makefile.am,v 1.164 2021/01/09 15:09:58 christos Exp $
+# $File: Makefile.am,v 1.165 2021/02/05 21:54:04 christos Exp $
 #
 MAGIC_FRAGMENT_BASE = Magdir
 MAGIC_DIR = $(top_srcdir)/magic
@@ -46,6 +46,7 @@ $(MAGIC_FRAGMENT_DIR)/blackberry \
 $(MAGIC_FRAGMENT_DIR)/blcr \
 $(MAGIC_FRAGMENT_DIR)/blender \
 $(MAGIC_FRAGMENT_DIR)/blit \
+$(MAGIC_FRAGMENT_DIR)/bm \
 $(MAGIC_FRAGMENT_DIR)/bout \
 $(MAGIC_FRAGMENT_DIR)/bsdi \
 $(MAGIC_FRAGMENT_DIR)/bsi \

src/apprentice.c

@@ -32,7 +32,7 @@
 #include "file.h"
 
 #ifndef	lint
-FILE_RCSID("@(#)$File: apprentice.c,v 1.299 2020/12/16 23:38:39 christos Exp $")
+FILE_RCSID("@(#)$File: apprentice.c,v 1.300 2021/02/05 21:33:49 christos Exp $")
 #endif	/* lint */
 
 #include "magic.h"
@@ -543,6 +543,7 @@ file_ms_alloc(int flags)
 	ms->elf_notes_max = FILE_ELF_NOTES_MAX;
 	ms->regex_max = FILE_REGEX_MAX;
 	ms->bytes_max = FILE_BYTES_MAX;
+	ms->encoding_max = FILE_ENCODING_MAX;
 	return ms;
 free:
 	free(ms);

src/encoding.c

@@ -35,7 +35,7 @@
 #include "file.h"
 
 #ifndef	lint
-FILE_RCSID("@(#)$File: encoding.c,v 1.26 2020/08/15 12:06:07 christos Exp $")
+FILE_RCSID("@(#)$File: encoding.c,v 1.27 2021/02/05 21:33:49 christos Exp $")
 #endif	/* lint */
 
 #include "magic.h"
@@ -94,6 +94,9 @@ file_encoding(struct magic_set *ms, const struct buffer *b,
 	*code = "unknown";
 	*code_mime = "binary";
 
+	if (nbytes > ms->encoding_max)
+		nbytes = ms->encoding_max;
+
 	mlen = (nbytes + 1) * sizeof((*ubuf)[0]);
 	*ubuf = CAST(file_unichar_t *, calloc(CAST(size_t, 1), mlen));
 	if (*ubuf == NULL) {

src/file.c

@@ -32,7 +32,7 @@
 #include "file.h"
 
 #ifndef	lint
-FILE_RCSID("@(#)$File: file.c,v 1.188 2020/12/06 14:58:50 christos Exp $")
+FILE_RCSID("@(#)$File: file.c,v 1.189 2021/02/05 21:33:49 christos Exp $")
 #endif	/* lint */
 
 #include "magic.h"
@@ -146,6 +146,8 @@ private struct {
 	    "max ELF prog sections processed" },
 	{ "elf_shnum",	MAGIC_PARAM_ELF_SHNUM_MAX, 0, 0, FILE_ELF_SHNUM_MAX,
 	    "max ELF sections processed" },
+	{ "encoding",	MAGIC_PARAM_ENCODING_MAX, 0, 0, FILE_ENCODING_MAX,
+	    "max bytes to scan for encoding" },
 	{ "indir",	MAGIC_PARAM_INDIR_MAX, 0, 0, FILE_INDIR_MAX,
 	    "recursion limit for indirection" },
 	{ "name",	MAGIC_PARAM_NAME_MAX, 0, 0, FILE_NAME_MAX,

src/file.h

@@ -27,7 +27,7 @@
  */
 /*
  * file.h - definitions for file(1) program
- * @(#)$File: file.h,v 1.223 2020/12/08 21:26:00 christos Exp $
+ * @(#)$File: file.h,v 1.224 2021/02/05 21:33:49 christos Exp $
  */
 
 #ifndef __file_h__
@@ -462,6 +462,7 @@ struct magic_set {
 	uint16_t elf_notes_max;
 	uint16_t regex_max;
 	size_t bytes_max;		/* number of bytes to read from file */
+	size_t encoding_max;		/* bytes to look for encoding */
 #ifndef FILE_BYTES_MAX
 # define FILE_BYTES_MAX (1024 * 1024)	/* how much of the file to look at */
 #endif
@@ -471,6 +472,7 @@ struct magic_set {
 #define	FILE_INDIR_MAX			50
 #define	FILE_NAME_MAX			50
 #define	FILE_REGEX_MAX			8192
+#define	FILE_ENCODING_MAX		(64 * 1024)
 };
 
 /* Type for Unicode characters */

src/magic.c

@@ -33,7 +33,7 @@
 #include "file.h"
 
 #ifndef	lint
-FILE_RCSID("@(#)$File: magic.c,v 1.113 2020/12/08 21:26:00 christos Exp $")
+FILE_RCSID("@(#)$File: magic.c,v 1.114 2021/02/05 21:33:49 christos Exp $")
 #endif	/* lint */
 
 #include "magic.h"
@@ -617,6 +617,9 @@ magic_setparam(struct magic_set *ms, int param, const void *val)
 	case MAGIC_PARAM_BYTES_MAX:
 		ms->bytes_max = *CAST(const size_t *, val);
 		return 0;
+	case MAGIC_PARAM_ENCODING_MAX:
+		ms->encoding_max = *CAST(const size_t *, val);
+		return 0;
 	default:
 		errno = EINVAL;
 		return -1;
@@ -650,6 +653,9 @@ magic_getparam(struct magic_set *ms, int param, void *val)
 	case MAGIC_PARAM_BYTES_MAX:
 		*CAST(size_t *, val) = ms->bytes_max;
 		return 0;
+	case MAGIC_PARAM_ENCODING_MAX:
+		*CAST(size_t *, val) = ms->encoding_max;
+		return 0;
 	default:
 		errno = EINVAL;
 		return -1;

src/memtest.c

+/*
+ * Copyright (c) Christos Zoulas 2021.
+ * All Rights Reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice immediately at the beginning of the file, without modification,
+ *    this list of conditions, and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR
+ * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <sys/mman.h>
+#include <stdio.h>
+#include <stdbool.h>
+#include <string.h>
+#include <stdlib.h>
+#include <err.h>
+#include <fcntl.h>
+#include <unistd.h>
+#include <dlfcn.h>
+#include <magic.h>
+
+void *
+malloc(size_t len)
+{
+	char buf[512];
+	void *(*orig)(size_t) = dlsym(RTLD_NEXT, "malloc");
+	void *p = (*orig)(len);
+	int l = snprintf(buf, sizeof(buf), "malloc %zu %p\n", len, p);
+	write(2, buf, l);
+	return p;
+}
+
+void
+free(void *p)
+{
+	char buf[512];
+	void (*orig)(void *) = dlsym(RTLD_NEXT, "free");
+	(*orig)(p);
+	int l = snprintf(buf, sizeof(buf), "free %p\n", p);
+	write(2, buf, l);
+}
+
+void *
+calloc(size_t len, size_t nitems)
+{
+	char buf[512];
+	void *(*orig)(size_t, size_t) = dlsym(RTLD_NEXT, "calloc");
+	void *p = (*orig)(len, nitems);
+	size_t tot = len * nitems;
+	int l = snprintf(buf, sizeof(buf), "calloc %zu %p\n", tot, p);
+	write(2, buf, l);
+	return p;
+}
+void *
+realloc(void *q, size_t len)
+{
+	char buf[512];
+	void *(*orig)(void *, size_t) = dlsym(RTLD_NEXT, "realloc");
+	void *p = (*orig)(q, len);
+	int l = snprintf(buf, sizeof(buf), "realloc %zu %p\n", len, p);
+	write(2, buf, l);
+	return p;
+}
+
+static void
+usage(void)
+{
+	fprintf(stderr, "Usage: test [-b] <filename>\n");
+	exit(EXIT_FAILURE);
+}
+
+int
+main(int argc, char *argv[])
+{
+	bool buf = false;
+	int c;
+
+	while ((c = getopt(argc, argv, "b")) != -1)
+		switch (c) {
+		case 'b':
+			buf = true;
+			break;
+		default:
+			usage();
+		}
+
+	argc -= optind;
+	argv += optind;
+
+	if (argc == 0)
+		usage();
+
+	magic_t m = magic_open(0);
+	if (m == NULL)
+		err(EXIT_FAILURE, "magic_open");
+
+	magic_load(m, NULL);
+
+	const char *r;
+	if (buf) {
+		int fd = open(argv[0], O_RDONLY);
+		if (fd == -1)
+			err(EXIT_FAILURE, "Cannot open `%s'", argv[0]);
+
+		struct stat st;
+		if (fstat(fd, &st) == -1)
+			err(EXIT_FAILURE, "Cannot stat `%s'", argv[0]);
+		size_t l = (size_t)st.st_size;
+		void *p = mmap(NULL, l, PROT_READ, MAP_FILE | MAP_PRIVATE, fd,
+		    (off_t)0);
+		if (p == MAP_FAILED)
+			err(EXIT_FAILURE, "Cannot map `%s'", argv[0]);
+		close(fd);
+		r = magic_buffer(m, p, l);
+		munmap(p, l);
+	} else {
+		r = magic_file(m, argv[0]);
+	}
+	magic_close(m);
+
+	printf("%s\n", r ? r : "(null)");
+
+	return 0;
+}