insns: more instruction macroizing/fixups; remote FUTURE tags

Add more instruction macros and fix problems. Adjust some matching
problems.

Remove all FUTURE tags from the instruction list, and add a bunch of
new CPUID tags. Hopefully a small step toward actually getting CPU
feature selection working properly in the future.

Signed-off-by: H. Peter Anvin <hpa@zytor.com>

asm/assemble.c

@@ -119,17 +119,7 @@ static int process_ea(operand *input, int rfield, opflags_t rflags,
                       insn *ins, enum ea_type expected,
                       const char **errmsgp);
 
-/*
+/* Convert a prefix to a byte value */
- * Return the byte value of a legacy prefix (possibly depending on context)
- * Returns one of the following values if the prefix has no byte value:
- */
-enum prefix_err {
-    PFE_NULL  = -1,             /* No output */
-    PFE_MULTI = -2,             /* Multibyte output (VEX, EVEX, REX2) */
-    PFE_ERR   = -3,             /* Invalid prefix use */
-    PFE_WHAT  = -4              /* Not a valid prefix (internal error) */
-};
-
 static int prefix_byte(enum prefixes pfx, const int bits);
 
 /*
@@ -1439,8 +1429,7 @@ static int64_t calcsize(insn *ins, const struct itemplate * const temp)
     bool lockcheck = true;
     enum reg_enum mib_index = R_none;   /* For a separate index reg form */
     const char *errmsg;
-    int need_byte;
+    int need_pfx[MAXPREFIX];
-    enum prefixes need_pfx;
 
     ins->rex     = 0;           /* Ensure REX is reset */
     ins->evex    = 0;		/* Ensure EVEX is reset */
@@ -1453,6 +1442,8 @@ static int64_t calcsize(insn *ins, const struct itemplate * const temp)
     /* Default operand size */
     ins->op_size = bits != 16 ? 32 : 16;
 
+    nasm_zero(need_pfx);
+
     while (*codes) {
         c = *codes++;
         op1 = (c & 3) + ((opex & 1) << 2);
@@ -1758,35 +1749,16 @@ static int64_t calcsize(insn *ins, const struct itemplate * const temp)
             break;
 
         case 0331:
-            need_byte = PFE_NULL;
+            ins->need_pfx[PPS_REP] = PFE_NULL;
-            need_pfx  = P_none;
+            break;
-            goto check_rep;
 
         case 0332:
-            need_byte = 0xf2;
+            ins->need_pfx[PPS_REP] = 0xf2;
-            need_pfx  = P_REPNE;
+            break;
-            goto check_rep;
 
         case 0333:
-            need_byte = 0xf3;
+            ins->need_pfx[PPS_REP] = 0xf3;
-            need_pfx  = P_REP;
-            goto check_rep;
-
-        check_rep:
-        {
-            const enum prefixes pfx = ins->prefixes[PPS_REP];
-            const int byte = prefix_byte(pfx, bits);
-            if (byte != need_byte) {
-                if (pfx) {
-                    nasm_warn(ERR_PASS2,
-                              "%s prefix invalid for this instruction",
-                              prefix_name(pfx));
-                } else {
-                    ins->prefixes[PPS_REP] = need_pfx;
-                }
-            }
             break;
-        }
 
         case 0334:
             ins->rex |= REX_L;  /* Ignored in 64-bit mode */
@@ -1856,19 +1828,27 @@ static int64_t calcsize(insn *ins, const struct itemplate * const temp)
             break;
 
         case 0360:
+            ins->need_pfx[PPS_REP]   = PFE_NULL;
+            ins->need_pfx[PPS_OSIZE] = PFE_NULL;
             break;
 
         case 0361:
-            length++;
+            ins->need_pfx[PPS_REP]   = PFE_NULL;
+            /* fall through */
+        case 0366:
+            ins->need_pfx[PPS_OSIZE] = 0x66;
             break;
 
         case 0364:
-        case 0365:
+            ins->need_pfx[PPS_OSIZE] = PFE_NULL;
+            break;
+
+        case 0365:
+            ins->need_pfx[PPS_ASIZE] = PFE_NULL;
             break;
 
-        case 0366:
         case 0367:
-            length++;
+            ins->need_pfx[PPS_ASIZE] = 0x67;
             break;
 
         case 0370:
@@ -2297,8 +2277,36 @@ static int emit_prefixes(struct out_data *data, const insn *ins)
 
     for (j = 0; j < MAXPREFIX; j++) {
         const enum prefixes pfx = ins->prefixes[j];
+        const int r = ins->need_pfx[j];
         int c = prefix_byte(pfx, bits);
 
+        if (r && r != c && !(r <= 0 && c <= 0)) {
+            switch (pfx) {
+            case P_none:
+            case P_O16:
+            case P_O32:
+            case P_O64:
+            case P_A16:
+            case P_A32:
+            case P_A64:
+                /* In these cases, allow the instruction to simply override */
+                c = r;
+                break;
+            default:
+                /*!
+                 *!prefix-invalid [on] invalid prefix for instruction
+                 *!  this instruction is only valid with certain combinations
+                 *!  of prefixes. The prefix will still be generated as
+                 *!  requested, but the result may be a completely different
+                 *!  instruction.
+                 */
+                nasm_warn(WARN_PREFIX_INVALID|ERR_PASS2,
+                          "invalid prefix %s for instruction, result may be unexpected",
+                          prefix_name(pfx));
+                break;
+            }
+        }
+
 	/* Various warnings and error conditions */
         switch ((int)pfx) {
         case R_ES:
@@ -2698,7 +2706,6 @@ static void gencode(struct out_data *data, insn *ins)
             break;
 
         case 0361:
-            out_rawbyte(data, 0x66);
             break;
 
         case 0364:
@@ -2707,7 +2714,6 @@ static void gencode(struct out_data *data, insn *ins)
 
         case 0366:
         case 0367:
-            out_rawbyte(data, c - 0366 + 0x66);
             break;
 
         case3(0370):

include/nasm.h

@@ -639,6 +639,7 @@ enum vex_class {
  */
 enum prefixes { /* instruction prefixes */
     P_none = 0,
+    P_any  = 1,
     PREFIX_ENUM_START = REG_ENUM_LIMIT,
     P_A16 = PREFIX_ENUM_START,
     P_A32,
@@ -858,10 +859,21 @@ enum optimization {
     OPTIM_DEFAULT = OPTIM_ALL_ENABLED
 };
 
+/*
+ * Used with byte values for prefixes when no single byte value applies
+ */
+enum prefix_err {
+    PFE_NULL  = -1,             /* No output */
+    PFE_MULTI = -2,             /* Multibyte output (VEX, EVEX, REX2) */
+    PFE_ERR   = -3,             /* Invalid prefix use */
+    PFE_WHAT  = -4              /* Not a valid prefix (internal error) */
+};
+
 typedef struct insn { /* an instruction itself */
     enum opcode     opcode;                 /* the opcode - not just the string */
     struct location loc;                    /* assembly location */
     int             prefixes[MAXPREFIX];    /* instruction prefixes, if any */
+    int             need_pfx[MAXPREFIX];    /* byte prefixes used as opcodes */
     char            *label;                 /* the label defined, or NULL */
     extop           *eops;                  /* extended operands */
     int             eops_float;             /* true if DD and floating */

travis/test/br3392252.asm

@@ -1,4 +1,5 @@
-[BITS 64]
+	bits 64
+	[warning -obsolete]
 
 	bextr rax, rsi, 1
 	bextr eax, esi, 1

x86/bytecode.txt

@@ -117,8 +117,8 @@ t = 0 for VEX (C4/C5), t = 1 for XOP (8F).
 \322             odf                         indicates that this instruction is only valid when the
                                              operand size is the default (instruction to disassembler,
                                              generates no code in the assembler)
-\323             o64nw*                      indicates fixed 64-bit operand size (equivalent to nw o64)
+\323             o64nw                       indicates fixed 64-bit operand size (equivalent to nw o64)
-\324             o64*                        indicates 64-bit operand size requiring REX.W.
+\324             o64                         indicates 64-bit operand size requiring REX.W.
 \325             nohi                        instruction which always uses spl/bpl/sil/dil
 \326             nof3                        (disassembler only) not valid with 0xF3 REP prefix.
 \327		 nw			     indicates that the operand size defaults to 64 in 64-bit mode;
@@ -131,8 +131,8 @@ t = 0 for VEX (C4/C5), t = 1 for XOP (8F).
 					           with code rex.w (\347).
 \330		 osz			     treated as o16 or o32 depending on the current mode.
 \331             norep                       not valid with 0xF2 or 0xF3 REP prefixes.
-\332             f2i                         REP prefix (0xF2 byte) used as opcode extension.
+\332             f2                          REP prefix (0xF2 byte) used as opcode extension.
-\333             f3i                         REP prefix (0xF3 byte) used as opcode extension.
+\333             f3                          REP prefix (0xF3 byte) used as opcode extension.
 \334             rex.l                       LOCK prefix used as REX.R in 16/32-bit mode.
 \335             repe                        disassemble a rep (0xF3 byte) prefix as repe not rep.
 \340             resb                        reserve <operand 0> bytes of uninitialized storage.
@@ -150,11 +150,11 @@ t = 0 for VEX (C4/C5), t = 1 for XOP (8F).
 -		 m0			     Generates no byte code, but can be used to indicate that
 					     following bytes are literal and not part of a prefix.
 \360             np                          no SSE prefix (== \364\331)
-\361                                         66 SSE prefix (== \366\331)
+\361             66                          66 SSE prefix (== \366\331)
 \364             !osp                        operand-size prefix (0x66) not permitted
 \365             !asp                        address-size prefix (0x67) not permitted
-\366                                         operand-size prefix (0x66) used as opcode extension
+\366             osp                         operand-size prefix (0x66) used as opcode extension
-\367                                         address-size prefix (0x67) used as opcode extension
+\367             67                          address-size prefix (0x67) used as opcode extension
 \370,\371        jcc8                        match only if operand 0 meets byte jump criteria.
                  jmp8                        370 is used for Jcc, 371 is used for JMP.
 \373             jlen                        assemble 0x03 if bits==16, 0x05 if bits==32;
@@ -163,4 +163,4 @@ t = 0 for VEX (C4/C5), t = 1 for XOP (8F).
 \375             vsiby|vm32y|vm64y           this instruction takes an YMM VSIB memory EA
 \376             vsibz|vm32z|vm64z           this instruction takes an ZMM VSIB memory EA
 
-* No prefix is emitted if following VEX/EVEX.
+* No 66 prefix is emitted if combined with VEX/EVEX, np, 66, osp or !osp.

x86/iflags.ph

@@ -98,19 +98,18 @@ if_("SSE42",             "SSE4.2");
 if_("SSE5",              "SSE5");
 if_("AVX",               "AVX  (256-bit floating point)");
 if_("AVX2",              "AVX2 (256-bit integer)");
-if_("FMA",               "");
+if_("FMA",               "Fused multiply-add");
-if_("BMI1",              "");
+if_("BMI1",              "Bit manipulation instructions 1");
-if_("BMI2",              "");
+if_("BMI2",              "Bit manipulation instructions 2");
 if_("TBM",               "");
 if_("RTM",               "");
-if_("INVPCID",           "");
+if_("AVX512",            "AVX-512");
-if_("AVX512",            "AVX-512F (512-bit base architecture)");
+if_("AVX512F",           "AVX-512F (base architecture)");
 if_("AVX512CD",          "AVX-512 Conflict Detection");
 if_("AVX512ER",          "AVX-512 Exponential and Reciprocal");
 if_("AVX512PF",          "AVX-512 Prefetch");
 if_("MPX",               "MPX");
 if_("SHA",               "SHA");
-if_("PREFETCHWT1",       "PREFETCHWT1");
 if_("AVX512VL",          "AVX-512 Vector Length Orthogonality");
 if_("AVX512DQ",          "AVX-512 Dword and Qword");
 if_("AVX512BW",          "AVX-512 Byte and Word");
@@ -131,22 +130,17 @@ if_("AVX512FC16",        "AVX-512 FC16 instructions");
 if_("SGX",               "Intel Software Guard Extensions (SGX)");
 if_("CET",               "Intel Control-Flow Enforcement Technology (CET)");
 if_("ENQCMD",            "Enqueue command instructions");
-if_("PCONFIG",           "Platform configuration instruction");
-if_("WBNOINVD",          "Writeback and do not invalidate instruction");
 if_("TSXLDTRK",          "TSX suspend load address tracking");
-if_("SERIALIZE",         "SERIALIZE instruction");
 if_("AVX512BF16",        "AVX-512 bfloat16");
 if_("AVX512VP2INTERSECT", "AVX-512 VP2INTERSECT instructions");
 if_("AMXTILE",           "AMX tile configuration instructions");
 if_("AMXBF16",           "AMX bfloat16 multiplication");
 if_("AMXINT8",           "AMX 8-bit integer multiplication");
 if_("FRED",              "Flexible Return and Exception Delivery (FRED)");
-if_("LKGS",              "Load User GS from Kernel (LKGS)");
 if_("RAOINT",		 "Remote atomic operations (RAO-INT)");
 if_("UINTR",		 "User interrupts");
 if_("CMPCCXADD",         "CMPccXADD instructions");
 if_("PREFETCHI",         "PREFETCHI0 and PREFETCHI1");
-if_("WRMSRNS",		 "WRMSRNS");
 if_("MSRLIST",           "RDMSRLIST and WRMSRLIST");
 if_("AVXNECONVERT",	 "AVX exceptionless floating-point conversions");
 if_("AVXVNNIINT8",       "AVX Vector Neural Network 8-bit integer instructions");
@@ -159,6 +153,16 @@ if_("HSM4",              "SM4 hash instructions");
 if_("APX",               "Advanced Performance Extensions (APX)");
 if_("AVX10_1",           "AVX 10.1 instructions");
 if_("AVX10_2",           "AVX 10.2 instructions");
+if_("ADX",               "ADCX and ADOX instructions");
+if_("PKU",		 "Protection key for user mode");
+if_("WAITPKG",           "User wait instruction package");
+
+# Single-instruction CPUID bits without additional help text
+foreach my $ins (qw(invpcid prefetchwt1 pconfig wbnoinvd serialize lkgs
+		    wrmsrns clflushopt clwb rdrand rdseed rdpid
+		    lzcnt ptwrite cldemote movdiri movdir64b)) {
+    if_($ins, "\U$ins\E instruction");
+}
 
 # Put these last to minimize their relevance
 if_("OBSOLETE",          "Instruction removed from architecture");
@@ -183,12 +187,12 @@ if_("P6",                "P6");
 if_("KATMAI",            "Katmai");
 if_("WILLAMETTE",        "Willamette");
 if_("PRESCOTT",          "Prescott");
+if_("IA64",              "IA64 (in x86 mode)");
 if_("X86_64",            "x86-64 (long or legacy mode)");
 if_("NEHALEM",           "Nehalem");
 if_("WESTMERE",          "Westmere");
 if_("SANDYBRIDGE",       "Sandy Bridge");
 if_("FUTURE",            "Ivy Bridge or newer");
-if_("IA64",              "IA64 (in x86 mode)");
 
 # Default CPU level
 if_("DEFAULT",           "Default CPU level");

x86/insns-iflags.ph

@@ -87,6 +87,12 @@ our $MAX_OPERANDS = 5;
 sub if_($$) {
     my($name, $def) = @_;
     my $num = $n_iflags++;
+
+    $name = uc($name);
+    if (defined($flag_byname{$name})) {
+	die "iflags: flag $name defined more than once\n";
+    }
+
     my $v = [$num, $name, $def];
 
     if (!($n_iflags & 31) && $no_word_break) {
@@ -178,6 +184,7 @@ sub set_implied_flags($;$) {
     $flags->{'ZU'}++ if ($flags->{'ZU_R'} || $flags->{'ZU_E'});
 
     # Retain only the highest CPU level flag
+    # CPU levels really need to be replaced with feature sets.
     my $found = 0;
     for (my $i = $flag_byname{'ANY'}->[0]; $i >= $flag_byname{'8086'}->[0]; $i--) {
 	my $f = $flag_bynum[$i]->[1];
@@ -187,6 +194,10 @@ sub set_implied_flags($;$) {
 	    $found = $flags->{$f};
 	}
     }
+    if (!$found) {
+	# No CPU level flag at all; tag it FUTURE
+	$flags->{'FUTURE'}++;
+    }
 }
 
 # Return the value of any assume-size flag if one exists;

x86/insns.pl

@@ -997,6 +997,7 @@ sub byte_code_compile($$$) {
         'a64'       => 0313,
         '!osp'      => 0364,
         '!asp'      => 0365,
+	'osp'       => 0366,
 	'osz'       => 0330,	# 66 or REX.W if operand size != default
         'f2i'       => 0332,    # F2 prefix, but 66 for operand size is OK
         'f3i'       => 0333,    # F3 prefix, but 66 for operand size is OK
@@ -1142,6 +1143,8 @@ sub byte_code_compile($$$) {
             my ($m,$w,$l,$p) = (undef,undef,undef,0);
             my $has_nds = 0;
             my @subops = split(/\./, $2);
+	    my $opsize = undef;
+
 	    if (defined($opmap)) {
 		warn "$fname:$line: $opcode: legacy prefix ignored with VEX\n";
 	    }
@@ -1165,6 +1168,31 @@ sub byte_code_compile($$$) {
 		} elsif ($oq eq 'ww') {
 		    $w = 0;
 		    $flags->{'WW'}++;
+		} elsif ($oq eq 'o8') {
+		    $p = 0 unless (defined($p)); # np
+		    if (!defined($w)) {		 # wig
+			$w = 0;
+			$flags->{'WIG'}++;
+		    }
+		} elsif ($oq eq 'o16') {
+		    $p = 1 unless (defined($p)); # 66
+		    $w = 0 unless (defined($w)); # w0
+		    $opsize = 0320;
+		} elsif ($oq eq 'ko8') {
+		    $p = 1 unless (defined($p)); # 66
+		    $w = 0 unless (defined($w)); # w0
+		} elsif ($oq eq 'ko16') {
+		    $p = 1 unless (defined($p)); # 66
+		    $w = 1 unless (defined($w)); # w1
+		    $opsize = 0320;
+		} elsif ($oq =~ /^k?o32$/) {
+		    $p = 0 unless (defined($p)); # np
+		    $w = 0 unless (defined($w)); # w0
+		    $opsize = 0321;
+		} elsif ($oq =~ /^k?o64$/) {
+		    $p = 0 unless (defined($p)); # np
+		    $w = 1 unless (defined($w)); # w1
+		    $opsize = 0323 + $w;
 		} elsif ($oq eq 'np' || $oq eq 'p0') {
 		    $p = 0;
 		} elsif ($oq eq '66' || $oq eq 'p1') {
@@ -1207,6 +1235,8 @@ sub byte_code_compile($$$) {
             push(@codes, defined($oppos{'v'}) ? 0260+$oppos{'v'} : 0270,
                  ($c << 6)+$m, ($w << 7)+($l << 2)+$p);
 
+	    push(@codes, $opsize) if (defined($opsize));
+
 	    $flags->{'VEX'}++;
 	    $flags->{'NOAPX'}++; # VEX doesn't support registers 16+
             $prefix_ok = 0;
@@ -1263,11 +1293,18 @@ sub byte_code_compile($$$) {
 		    $p = 1 unless (defined($p)); # 66
 		    $w = 0 unless (defined($w)); # w0
 		    $opsize = 0320;
-		} elsif ($oq eq 'o32') {
+		} elsif ($oq eq 'ko8') {
+		    $p = 1 unless (defined($p)); # 66
+		    $w = 0 unless (defined($w)); # w0
+		} elsif ($oq eq 'ko16') {
+		    $p = 1 unless (defined($p)); # 66
+		    $w = 1 unless (defined($w)); # w1
+		    $opsize = 0320;
+		} elsif ($oq =~ /^k?o32$/) {
 		    $p = 0 unless (defined($p)); # np
 		    $w = 0 unless (defined($w)); # w0
 		    $opsize = 0321;
-		} elsif ($oq eq 'o64') {
+		} elsif ($oq =~ /^k?o64$/) {
 		    $p = 0 unless (defined($p)); # np
 		    $w = 1 unless (defined($w)); # w1
 		    $opsize = 0323 + $w;

x86/preinsns.pl

@@ -79,7 +79,7 @@ sub func_multisize($$$) {
 	my $sn = $sizename[$i];
 	my $sz = $s || 'sz';
 	my $o;
-	my $ins = join("\t", @$rawargs);
+	my $ins = join(' ', @$rawargs);
 
 	# Conditional pattern inclusions
 	# Syntax: (which1:text1/which2:text2/text3)
@@ -108,16 +108,26 @@ sub func_multisize($$$) {
 	$ins = $o.$ins;
 	$o = '';
 
-	while ($ins =~ /^(.*?)((?:\b[0-9a-f]{2}(?:\+r)?|\bsbyte|\bimm|\b[ioa]|\b(?:reg_)?[abcd]x|\breg|\brm|\bw)?\#|\b(?:reg|rm)64\b|\b(?:o64)?nw\b|\b(?:NO)?LONG\w+\b|\%)(.*)$/) {
+	print '>', $ins, "\n";
+
+	while ($ins =~ /^(.*?)((?:\b[0-9a-f]{2}(?:\+r)?|\bsbyte|\bimm|\b[ioa]|\b(?:reg_)?[abcd]x|\breg|\brm|\bw)?\#{1,2}|\b(?:reg|rm)64\b|\b(?:o64)?nw\b|\b(?:NO)?LONG\w+\b|\%{1,2})(.*)$/) {
 	    $o .= $1;
 	    my $mw = $2;
 	    $ins = $3;
-	    if ($mw eq '%') {
+	    if (!$i && $mw =~ /\#\#$/) {
+		die "$0:$infile:$line: $mw cannot be used with z\n";
+	    } elsif ($mw eq '%') {
 		$o .= uc($sn) if ($i);
-	    } elsif ($mw =~ /^([0-9a-f]{2})(\+r)?\#$/) {
+	    } elsif ($mw eq '%%') {
-		$o .= sprintf('%02x%s', hex($1) |
+		if ($i < 2) {
-			      (($s == 8) ? 0 :
+		    die "$0:$infile:$line: $mw cannot be used with zb\n";
-			       ($2 eq '') ? 1 : 8), $2);
+		}
+		$o .= uc($sizename[$i-1]) . uc($sn);
+	    } elsif ($mw =~ /^([0-9a-f]{2})(\+r)?(\#)?\#$/) {
+		my $n;
+		$n = ($3 ne '') ? $s > 16 : $s > 8;
+		$n <<= 3 if ($2 ne '');
+		$o .= sprintf('%02x%s', hex($1) | $n, $2);
 	    } elsif ($mw eq 'sbyte#') {
 		$o .= $sbyte[$i];
 	    } elsif ($mw eq 'imm#') {
@@ -154,6 +164,8 @@ sub func_multisize($$$) {
 		# Drop
 	    } elsif ($mw eq 'w#') {
 		$o .= !$i ? 'ww' : ($s >= 64) ? 'w1' : 'w0';
+	    } elsif ($mw eq 'w##') {
+		$o .= 'w'.(($i-1) & 1);
 	    } elsif ($mw eq '#') {
 		$o .= $s;
 	    } else {
@@ -173,6 +185,54 @@ sub func_multisize($$$) {
     return @ol;
 }
 
+# Common pattern for K-register instructions
+$macros{'k'} = {
+    'func' =>
+	sub {
+	    my($mac, $args, $rawargs) = @_;
+	    my @ol;
+	    my $ins = join(' ', @$rawargs);
+	    my $xins;
+	    my $n;
+
+	    $ins .= ',ZU';
+	    ($xins = $ins) =~ s/\bSM[0-9-]*,?//;
+	    push(@ol, $xins);
+	    ($xins = $ins) =~ s/\%//;
+	    push(@ol, $xins);
+	    if ($xins =~ s/\%//) {
+		push(@ol, $xins);
+	    }
+
+	    # Allow instruction without K
+	    my @on;
+	    foreach my $oi (@ol) {
+		# Remove first capital K
+		($xins = $oi) =~ s/\bK//;
+		push(@on, $xins);
+	    }
+	    push(@ol, @on);
+	    undef @on;
+
+	    # Allow SHIFT -> SH
+	    if ($ins =~ /SHIFT/) {
+		foreach my $oi (@ol) {
+		    # Remove first capital K
+		    ($xins = $oi) =~ s/SHIFT/SH/;
+		    push(@on, $xins);
+		}
+	    }
+	    push(@ol, @on);
+	    undef @on;
+
+	    # All instruction patterns except the first are ND
+	    for (my $i = 1; $i < scalar(@ol); $i++) {
+		$ol[$i] .= ',ND';
+	    }
+	    return(@ol);
+    }
+};
+
 # Common pattern for the HINT_NOPx pseudo-instructions
 $macros{'hint_nops'} = {
     'func' =>