st_glsl_to_tgsi.cpp 172 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29
/*
 * Copyright (C) 2005-2007  Brian Paul   All Rights Reserved.
 * Copyright (C) 2008  VMware, Inc.   All Rights Reserved.
 * Copyright © 2010 Intel Corporation
 * Copyright © 2011 Bryan Cain
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the "Software"),
 * to deal in the Software without restriction, including without limitation
 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
 * and/or sell copies of the Software, and to permit persons to whom the
 * Software is furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice (including the next
 * paragraph) shall be included in all copies or substantial portions of the
 * Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
 * DEALINGS IN THE SOFTWARE.
 */

/**
 * \file glsl_to_tgsi.cpp
 *
30
 * Translate GLSL IR to TGSI.
31 32 33 34 35 36 37 38 39 40 41 42 43 44 45
 */

#include <stdio.h>
#include "main/compiler.h"
#include "ir.h"
#include "ir_visitor.h"
#include "ir_expression_flattening.h"
#include "glsl_types.h"
#include "glsl_parser_extras.h"
#include "../glsl/program.h"
#include "ir_optimization.h"
#include "ast.h"

#include "main/mtypes.h"
#include "main/shaderobj.h"
46
#include "main/uniforms.h"
47
#include "program/hash_table.h"
48 49 50

extern "C" {
#include "main/shaderapi.h"
51 52 53 54 55 56 57 58 59 60 61 62 63 64
#include "program/prog_instruction.h"
#include "program/prog_optimize.h"
#include "program/prog_print.h"
#include "program/program.h"
#include "program/prog_parameter.h"
#include "program/sampler.h"

#include "pipe/p_compiler.h"
#include "pipe/p_context.h"
#include "pipe/p_screen.h"
#include "pipe/p_shader_tokens.h"
#include "pipe/p_state.h"
#include "util/u_math.h"
#include "tgsi/tgsi_ureg.h"
65
#include "tgsi/tgsi_info.h"
66 67 68 69
#include "st_context.h"
#include "st_program.h"
#include "st_glsl_to_tgsi.h"
#include "st_mesa_to_tgsi.h"
70
}
71

72
#define PROGRAM_IMMEDIATE PROGRAM_FILE_MAX
73
#define PROGRAM_ANY_CONST ((1 << PROGRAM_STATE_VAR) |    \
74 75 76
                           (1 << PROGRAM_CONSTANT) |     \
                           (1 << PROGRAM_UNIFORM))

77 78 79 80 81 82
/**
 * Maximum number of temporary registers.
 *
 * It is too big for stack allocated arrays -- it will cause stack overflow on
 * Windows and likely Mac OS X.
 */
83 84
#define MAX_TEMPS         4096

85 86 87 88 89
/**
 * Maximum number of arrays
 */
#define MAX_ARRAYS        256

90
/* if we support a native gallium TG4 with the ability to take 4 texoffsets then bump this */
91 92
#define MAX_GLSL_TEXTURE_OFFSET 1

93 94 95 96 97 98
class st_src_reg;
class st_dst_reg;

static int swizzle_for_size(int size);

/**
99
 * This struct is a corresponding struct to TGSI ureg_src.
100 101 102 103 104 105 106 107 108 109 110 111
 */
class st_src_reg {
public:
   st_src_reg(gl_register_file file, int index, const glsl_type *type)
   {
      this->file = file;
      this->index = index;
      if (type && (type->is_scalar() || type->is_vector() || type->is_matrix()))
         this->swizzle = swizzle_for_size(type->vector_elements);
      else
         this->swizzle = SWIZZLE_XYZW;
      this->negate = 0;
112
      this->index2D = 0;
113
      this->type = type ? type->base_type : GLSL_TYPE_ERROR;
114
      this->reladdr = NULL;
115 116
      this->reladdr2 = NULL;
      this->has_index2 = false;
117 118
   }

119
   st_src_reg(gl_register_file file, int index, int type)
120
   {
121
      this->type = type;
122 123
      this->file = file;
      this->index = index;
124 125 126 127
      this->index2D = 0;
      this->swizzle = SWIZZLE_XYZW;
      this->negate = 0;
      this->reladdr = NULL;
128 129
      this->reladdr2 = NULL;
      this->has_index2 = false;
130 131 132 133 134 135 136 137
   }

   st_src_reg(gl_register_file file, int index, int type, int index2D)
   {
      this->type = type;
      this->file = file;
      this->index = index;
      this->index2D = index2D;
138 139 140
      this->swizzle = SWIZZLE_XYZW;
      this->negate = 0;
      this->reladdr = NULL;
141 142
      this->reladdr2 = NULL;
      this->has_index2 = false;
143 144
   }

145 146
   st_src_reg()
   {
147
      this->type = GLSL_TYPE_ERROR;
148 149
      this->file = PROGRAM_UNDEFINED;
      this->index = 0;
150
      this->index2D = 0;
151 152 153
      this->swizzle = 0;
      this->negate = 0;
      this->reladdr = NULL;
154 155
      this->reladdr2 = NULL;
      this->has_index2 = false;
156 157 158 159 160
   }

   explicit st_src_reg(st_dst_reg reg);

   gl_register_file file; /**< PROGRAM_* from Mesa */
161
   int index; /**< temporary index, VERT_ATTRIB_*, VARYING_SLOT_*, etc. */
162
   int index2D;
163 164
   GLuint swizzle; /**< SWIZZLE_XYZWONEZERO swizzles from Mesa. */
   int negate; /**< NEGATE_XYZW mask from mesa */
165
   int type; /** GLSL_TYPE_* from GLSL IR (enum glsl_base_type) */
166 167
   /** Register index should be offset by the integer in this reg. */
   st_src_reg *reladdr;
168 169
   st_src_reg *reladdr2;
   bool has_index2;
170 171 172 173
};

class st_dst_reg {
public:
174 175 176 177 178 179 180 181 182 183
   st_dst_reg(gl_register_file file, int writemask, int type, int index)
   {
      this->file = file;
      this->index = index;
      this->writemask = writemask;
      this->cond_mask = COND_TR;
      this->reladdr = NULL;
      this->type = type;
   }

184
   st_dst_reg(gl_register_file file, int writemask, int type)
185 186 187 188 189 190
   {
      this->file = file;
      this->index = 0;
      this->writemask = writemask;
      this->cond_mask = COND_TR;
      this->reladdr = NULL;
191
      this->type = type;
192 193 194 195
   }

   st_dst_reg()
   {
196
      this->type = GLSL_TYPE_ERROR;
197 198 199 200 201 202 203 204 205 206
      this->file = PROGRAM_UNDEFINED;
      this->index = 0;
      this->writemask = 0;
      this->cond_mask = COND_TR;
      this->reladdr = NULL;
   }

   explicit st_dst_reg(st_src_reg reg);

   gl_register_file file; /**< PROGRAM_* from Mesa */
207
   int index; /**< temporary index, VERT_ATTRIB_*, VARYING_SLOT_*, etc. */
208 209
   int writemask; /**< Bitfield of WRITEMASK_[XYZW] */
   GLuint cond_mask:4;
210
   int type; /** GLSL_TYPE_* from GLSL IR (enum glsl_base_type) */
211 212 213 214 215 216
   /** Register index should be offset by the integer in this reg. */
   st_src_reg *reladdr;
};

st_src_reg::st_src_reg(st_dst_reg reg)
{
217
   this->type = reg.type;
218 219 220 221
   this->file = reg.file;
   this->index = reg.index;
   this->swizzle = SWIZZLE_XYZW;
   this->negate = 0;
222
   this->reladdr = reg.reladdr;
223
   this->index2D = 0;
224 225
   this->reladdr2 = NULL;
   this->has_index2 = false;
226 227 228 229
}

st_dst_reg::st_dst_reg(st_src_reg reg)
{
230
   this->type = reg.type;
231 232 233 234 235 236 237 238 239
   this->file = reg.file;
   this->index = reg.index;
   this->writemask = WRITEMASK_XYZW;
   this->cond_mask = COND_TR;
   this->reladdr = reg.reladdr;
}

class glsl_to_tgsi_instruction : public exec_node {
public:
240
   DECLARE_RALLOC_CXX_OPERATORS(glsl_to_tgsi_instruction)
241

242
   unsigned op;
243
   st_dst_reg dst;
244
   st_src_reg src[4];
245 246 247 248 249 250 251
   /** Pointer to the ir source this tree came from for debugging */
   ir_instruction *ir;
   GLboolean cond_update;
   bool saturate;
   int sampler; /**< sampler index */
   int tex_target; /**< One of TEXTURE_*_INDEX */
   GLboolean tex_shadow;
252 253

   st_src_reg tex_offsets[MAX_GLSL_TEXTURE_OFFSET];
254
   unsigned tex_offset_num_offset;
255
   int dead_mask; /**< Used in dead code elimination */
256

257
   class function_entry *function; /* Set on TGSI_OPCODE_CAL or TGSI_OPCODE_BGNSUB */
258 259 260 261 262 263 264 265 266 267 268 269 270 271 272
};

class variable_storage : public exec_node {
public:
   variable_storage(ir_variable *var, gl_register_file file, int index)
      : file(file), index(index), var(var)
   {
      /* empty */
   }

   gl_register_file file;
   int index;
   ir_variable *var; /* variable that maps to this, if any */
};

273 274 275 276 277 278 279 280 281 282 283 284 285 286
class immediate_storage : public exec_node {
public:
   immediate_storage(gl_constant_value *values, int size, int type)
   {
      memcpy(this->values, values, size * sizeof(gl_constant_value));
      this->size = size;
      this->type = type;
   }
   
   gl_constant_value values[4];
   int size; /**< Number of components (1-4) */
   int type; /**< GL_FLOAT, GL_INT, GL_BOOL, or GL_UNSIGNED_INT */
};

287 288 289 290 291 292 293
class function_entry : public exec_node {
public:
   ir_function_signature *sig;

   /**
    * identifier of this function signature used by the program.
    *
Bryan Cain's avatar
Bryan Cain committed
294
    * At the point that TGSI instructions for function calls are
295 296 297 298 299 300 301 302 303 304 305 306 307 308
    * generated, we don't know the address of the first instruction of
    * the function body.  So we make the BranchTarget that is called a
    * small integer and rewrite them during set_branchtargets().
    */
   int sig_id;

   /**
    * Pointer to first instruction of the function body.
    *
    * Set during function body emits after main() is processed.
    */
   glsl_to_tgsi_instruction *bgn_inst;

   /**
Bryan Cain's avatar
Bryan Cain committed
309
    * Index of the first instruction of the function body in actual TGSI.
310
    *
Bryan Cain's avatar
Bryan Cain committed
311
    * Set after conversion from glsl_to_tgsi_instruction to TGSI.
312 313 314 315 316 317 318
    */
   int inst;

   /** Storage for the return value. */
   st_src_reg return_reg;
};

319
struct glsl_to_tgsi_visitor : public ir_visitor {
320 321 322 323 324 325 326 327 328 329 330 331
public:
   glsl_to_tgsi_visitor();
   ~glsl_to_tgsi_visitor();

   function_entry *current_function;

   struct gl_context *ctx;
   struct gl_program *prog;
   struct gl_shader_program *shader_program;
   struct gl_shader_compiler_options *options;

   int next_temp;
332

333 334 335
   unsigned array_sizes[MAX_ARRAYS];
   unsigned next_array;

336
   int num_address_regs;
337
   int samplers_used;
338
   bool indirect_addr_consts;
339 340
   
   int glsl_version;
341
   bool native_integers;
342
   bool have_sqrt;
343 344 345

   variable_storage *find_variable_storage(ir_variable *var);

346 347 348
   int add_constant(gl_register_file file, gl_constant_value values[4],
                    int size, int datatype, GLuint *swizzle_out);

349 350 351 352 353 354
   function_entry *get_function_signature(ir_function_signature *sig);

   st_src_reg get_temp(const glsl_type *type);
   void reladdr_to_temp(ir_instruction *ir, st_src_reg *reg, int *num_reladdr);

   st_src_reg st_src_reg_for_float(float val);
355 356
   st_src_reg st_src_reg_for_int(int val);
   st_src_reg st_src_reg_for_type(int type, int val);
357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382

   /**
    * \name Visit methods
    *
    * As typical for the visitor pattern, there must be one \c visit method for
    * each concrete subclass of \c ir_instruction.  Virtual base classes within
    * the hierarchy should not have \c visit methods.
    */
   /*@{*/
   virtual void visit(ir_variable *);
   virtual void visit(ir_loop *);
   virtual void visit(ir_loop_jump *);
   virtual void visit(ir_function_signature *);
   virtual void visit(ir_function *);
   virtual void visit(ir_expression *);
   virtual void visit(ir_swizzle *);
   virtual void visit(ir_dereference_variable  *);
   virtual void visit(ir_dereference_array *);
   virtual void visit(ir_dereference_record *);
   virtual void visit(ir_assignment *);
   virtual void visit(ir_constant *);
   virtual void visit(ir_call *);
   virtual void visit(ir_return *);
   virtual void visit(ir_discard *);
   virtual void visit(ir_texture *);
   virtual void visit(ir_if *);
383 384
   virtual void visit(ir_emit_vertex *);
   virtual void visit(ir_end_primitive *);
385 386 387 388 389 390 391
   /*@}*/

   st_src_reg result;

   /** List of variable_storage */
   exec_list variables;

392 393
   /** List of immediate_storage */
   exec_list immediates;
394
   unsigned num_immediates;
395

396 397 398 399 400 401 402
   /** List of function_entry */
   exec_list function_signatures;
   int next_signature_id;

   /** List of glsl_to_tgsi_instruction */
   exec_list instructions;

403
   glsl_to_tgsi_instruction *emit(ir_instruction *ir, unsigned op);
404

405
   glsl_to_tgsi_instruction *emit(ir_instruction *ir, unsigned op,
406 407
        		        st_dst_reg dst, st_src_reg src0);

408
   glsl_to_tgsi_instruction *emit(ir_instruction *ir, unsigned op,
409 410
        		        st_dst_reg dst, st_src_reg src0, st_src_reg src1);

411
   glsl_to_tgsi_instruction *emit(ir_instruction *ir, unsigned op,
412 413
        		        st_dst_reg dst,
        		        st_src_reg src0, st_src_reg src1, st_src_reg src2);
414 415 416 417 418 419

   glsl_to_tgsi_instruction *emit(ir_instruction *ir, unsigned op,
                                  st_dst_reg dst,
                                  st_src_reg src0, st_src_reg src1,
                                  st_src_reg src2, st_src_reg src3);

420 421 422
   unsigned get_opcode(ir_instruction *ir, unsigned op,
                    st_dst_reg dst,
                    st_src_reg src0, st_src_reg src1);
423 424 425 426

   /**
    * Emit the correct dot-product instruction for the type of arguments
    */
427 428 429 430 431
   glsl_to_tgsi_instruction *emit_dp(ir_instruction *ir,
                                     st_dst_reg dst,
                                     st_src_reg src0,
                                     st_src_reg src1,
                                     unsigned elements);
432

433
   void emit_scalar(ir_instruction *ir, unsigned op,
434 435
        	    st_dst_reg dst, st_src_reg src0);

436
   void emit_scalar(ir_instruction *ir, unsigned op,
437 438
        	    st_dst_reg dst, st_src_reg src0, st_src_reg src1);

439 440
   void emit_arl(ir_instruction *ir, st_dst_reg dst, st_src_reg src0);

441
   void emit_scs(ir_instruction *ir, unsigned op,
442 443
        	 st_dst_reg dst, const st_src_reg &src);

444 445
   bool try_emit_mad(ir_expression *ir,
              int mul_operand);
446 447
   bool try_emit_mad_for_and_not(ir_expression *ir,
              int mul_operand);
448
   bool try_emit_sat(ir_expression *ir);
449 450 451 452 453

   void emit_swz(ir_expression *ir);

   bool process_move_condition(ir_rvalue *ir);

454
   void simplify_cmp(void);
455

456 457 458 459 460 461 462 463
   void rename_temp_register(int index, int new_index);
   int get_first_temp_read(int index);
   int get_first_temp_write(int index);
   int get_last_temp_read(int index);
   int get_last_temp_write(int index);

   void copy_propagate(void);
   void eliminate_dead_code(void);
464
   int eliminate_dead_code_advanced(void);
465 466 467
   void merge_registers(void);
   void renumber_registers(void);

468 469 470
   void emit_block_mov(ir_assignment *ir, const struct glsl_type *type,
                       st_dst_reg *l, st_src_reg *r);

471 472 473
   void *mem_ctx;
};

474
static st_src_reg undef_src = st_src_reg(PROGRAM_UNDEFINED, 0, GLSL_TYPE_ERROR);
475

476
static st_dst_reg undef_dst = st_dst_reg(PROGRAM_UNDEFINED, SWIZZLE_NOOP, GLSL_TYPE_ERROR);
477

478 479
static st_dst_reg address_reg = st_dst_reg(PROGRAM_ADDRESS, WRITEMASK_X, GLSL_TYPE_FLOAT, 0);
static st_dst_reg address_reg2 = st_dst_reg(PROGRAM_ADDRESS, WRITEMASK_X, GLSL_TYPE_FLOAT, 1);
480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508

static void
fail_link(struct gl_shader_program *prog, const char *fmt, ...) PRINTFLIKE(2, 3);

static void
fail_link(struct gl_shader_program *prog, const char *fmt, ...)
{
   va_list args;
   va_start(args, fmt);
   ralloc_vasprintf_append(&prog->InfoLog, fmt, args);
   va_end(args);

   prog->LinkStatus = GL_FALSE;
}

static int
swizzle_for_size(int size)
{
   int size_swizzles[4] = {
      MAKE_SWIZZLE4(SWIZZLE_X, SWIZZLE_X, SWIZZLE_X, SWIZZLE_X),
      MAKE_SWIZZLE4(SWIZZLE_X, SWIZZLE_Y, SWIZZLE_Y, SWIZZLE_Y),
      MAKE_SWIZZLE4(SWIZZLE_X, SWIZZLE_Y, SWIZZLE_Z, SWIZZLE_Z),
      MAKE_SWIZZLE4(SWIZZLE_X, SWIZZLE_Y, SWIZZLE_Z, SWIZZLE_W),
   };

   assert((size >= 1) && (size <= 4));
   return size_swizzles[size - 1];
}

509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529
static bool
is_tex_instruction(unsigned opcode)
{
   const tgsi_opcode_info* info = tgsi_get_opcode_info(opcode);
   return info->is_tex;
}

static unsigned
num_inst_dst_regs(unsigned opcode)
{
   const tgsi_opcode_info* info = tgsi_get_opcode_info(opcode);
   return info->num_dst;
}

static unsigned
num_inst_src_regs(unsigned opcode)
{
   const tgsi_opcode_info* info = tgsi_get_opcode_info(opcode);
   return info->is_tex ? info->num_src - 1 : info->num_src;
}

530
glsl_to_tgsi_instruction *
531
glsl_to_tgsi_visitor::emit(ir_instruction *ir, unsigned op,
532 533 534
                           st_dst_reg dst,
                           st_src_reg src0, st_src_reg src1,
                           st_src_reg src2, st_src_reg src3)
535 536 537
{
   glsl_to_tgsi_instruction *inst = new(mem_ctx) glsl_to_tgsi_instruction();
   int num_reladdr = 0, i;
538 539
   
   op = get_opcode(ir, op, dst, src0, src1);
540 541 542 543 544 545

   /* If we have to do relative addressing, we want to load the ARL
    * reg directly for one of the regs, and preload the other reladdr
    * sources into temps.
    */
   num_reladdr += dst.reladdr != NULL;
546 547 548
   num_reladdr += src0.reladdr != NULL || src0.reladdr2 != NULL;
   num_reladdr += src1.reladdr != NULL || src1.reladdr2 != NULL;
   num_reladdr += src2.reladdr != NULL || src2.reladdr2 != NULL;
549
   num_reladdr += src3.reladdr != NULL || src3.reladdr2 != NULL;
550

551
   reladdr_to_temp(ir, &src3, &num_reladdr);
552 553 554 555 556
   reladdr_to_temp(ir, &src2, &num_reladdr);
   reladdr_to_temp(ir, &src1, &num_reladdr);
   reladdr_to_temp(ir, &src0, &num_reladdr);

   if (dst.reladdr) {
557
      emit_arl(ir, address_reg, *dst.reladdr);
558 559 560 561 562 563 564 565 566
      num_reladdr--;
   }
   assert(num_reladdr == 0);

   inst->op = op;
   inst->dst = dst;
   inst->src[0] = src0;
   inst->src[1] = src1;
   inst->src[2] = src2;
567
   inst->src[3] = src3;
568
   inst->ir = ir;
569
   inst->dead_mask = 0;
570 571 572 573 574 575 576 577 578 579 580

   inst->function = NULL;
   
   /* Update indirect addressing status used by TGSI */
   if (dst.reladdr) {
      switch(dst.file) {
      case PROGRAM_STATE_VAR:
      case PROGRAM_CONSTANT:
      case PROGRAM_UNIFORM:
         this->indirect_addr_consts = true;
         break;
581 582 583
      case PROGRAM_IMMEDIATE:
         assert(!"immediates should not have indirect addressing");
         break;
584 585 586 587 588
      default:
         break;
      }
   }
   else {
589
      for (i=0; i<4; i++) {
590
         if(inst->src[i].reladdr) {
591
            switch(inst->src[i].file) {
592 593 594 595 596
            case PROGRAM_STATE_VAR:
            case PROGRAM_CONSTANT:
            case PROGRAM_UNIFORM:
               this->indirect_addr_consts = true;
               break;
597 598 599
            case PROGRAM_IMMEDIATE:
               assert(!"immediates should not have indirect addressing");
               break;
600 601 602 603 604 605 606 607
            default:
               break;
            }
         }
      }
   }

   this->instructions.push_tail(inst);
608

609 610 611
   return inst;
}

612 613 614 615 616 617 618
glsl_to_tgsi_instruction *
glsl_to_tgsi_visitor::emit(ir_instruction *ir, unsigned op,
                           st_dst_reg dst, st_src_reg src0,
                           st_src_reg src1, st_src_reg src2)
{
   return emit(ir, op, dst, src0, src1, src2, undef_src);
}
619 620

glsl_to_tgsi_instruction *
621
glsl_to_tgsi_visitor::emit(ir_instruction *ir, unsigned op,
622 623
        		 st_dst_reg dst, st_src_reg src0, st_src_reg src1)
{
624
   return emit(ir, op, dst, src0, src1, undef_src, undef_src);
625 626 627
}

glsl_to_tgsi_instruction *
628
glsl_to_tgsi_visitor::emit(ir_instruction *ir, unsigned op,
629 630 631
        		 st_dst_reg dst, st_src_reg src0)
{
   assert(dst.writemask != 0);
632
   return emit(ir, op, dst, src0, undef_src, undef_src, undef_src);
633 634 635
}

glsl_to_tgsi_instruction *
636
glsl_to_tgsi_visitor::emit(ir_instruction *ir, unsigned op)
637
{
638
   return emit(ir, op, undef_dst, undef_src, undef_src, undef_src, undef_src);
639 640
}

641 642 643 644 645 646 647 648 649 650
/**
 * Determines whether to use an integer, unsigned integer, or float opcode 
 * based on the operands and input opcode, then emits the result.
 */
unsigned
glsl_to_tgsi_visitor::get_opcode(ir_instruction *ir, unsigned op,
        		 st_dst_reg dst,
        		 st_src_reg src0, st_src_reg src1)
{
   int type = GLSL_TYPE_FLOAT;
651 652 653 654

   if (op == TGSI_OPCODE_MOV)
       return op;

655 656 657 658 659
   assert(src0.type != GLSL_TYPE_ARRAY);
   assert(src0.type != GLSL_TYPE_STRUCT);
   assert(src1.type != GLSL_TYPE_ARRAY);
   assert(src1.type != GLSL_TYPE_STRUCT);

660 661
   if (src0.type == GLSL_TYPE_FLOAT || src1.type == GLSL_TYPE_FLOAT)
      type = GLSL_TYPE_FLOAT;
662
   else if (native_integers)
663
      type = src0.type == GLSL_TYPE_BOOL ? GLSL_TYPE_INT : src0.type;
664 665 666

#define case4(c, f, i, u) \
   case TGSI_OPCODE_##c: \
667 668 669 670 671 672
      if (type == GLSL_TYPE_INT) \
         op = TGSI_OPCODE_##i; \
      else if (type == GLSL_TYPE_UINT) \
         op = TGSI_OPCODE_##u; \
      else \
         op = TGSI_OPCODE_##f; \
673
      break;
674

675 676 677
#define case3(f, i, u)  case4(f, f, i, u)
#define case2fi(f, i)   case4(f, f, i, i)
#define case2iu(i, u)   case4(i, LAST, i, u)
678 679 680 681 682 683 684 685 686 687 688 689 690

#define casecomp(c, f, i, u) \
   case TGSI_OPCODE_##c: \
      if (type == GLSL_TYPE_INT) \
         op = TGSI_OPCODE_##i; \
      else if (type == GLSL_TYPE_UINT) \
         op = TGSI_OPCODE_##u; \
      else if (native_integers) \
         op = TGSI_OPCODE_##f; \
      else \
         op = TGSI_OPCODE_##c; \
      break;

691 692 693 694 695 696 697 698
   switch(op) {
      case2fi(ADD, UADD);
      case2fi(MUL, UMUL);
      case2fi(MAD, UMAD);
      case3(DIV, IDIV, UDIV);
      case3(MAX, IMAX, UMAX);
      case3(MIN, IMIN, UMIN);
      case2iu(MOD, UMOD);
699 700 701 702 703 704

      casecomp(SEQ, FSEQ, USEQ, USEQ);
      casecomp(SNE, FSNE, USNE, USNE);
      casecomp(SGE, FSGE, ISGE, USGE);
      casecomp(SLT, FSLT, ISLT, USLT);

705
      case2iu(ISHR, USHR);
706 707 708

      case2fi(SSG, ISSG);
      case3(ABS, IABS, IABS);
709 710 711 712

      case2iu(IBFE, UBFE);
      case2iu(IMSB, UMSB);
      case2iu(IMUL_HI, UMUL_HI);
713 714 715 716 717 718 719
      default: break;
   }
   
   assert(op != TGSI_OPCODE_LAST);
   return op;
}

720
glsl_to_tgsi_instruction *
721 722 723 724
glsl_to_tgsi_visitor::emit_dp(ir_instruction *ir,
        		    st_dst_reg dst, st_src_reg src0, st_src_reg src1,
        		    unsigned elements)
{
725 726
   static const unsigned dot_opcodes[] = {
      TGSI_OPCODE_DP2, TGSI_OPCODE_DP3, TGSI_OPCODE_DP4
727 728
   };

729
   return emit(ir, dot_opcodes[elements - 2], dst, src0, src1);
730 731 732
}

/**
733
 * Emits TGSI scalar opcodes to produce unique answers across channels.
734
 *
735
 * Some TGSI opcodes are scalar-only, like ARB_fp/vp.  The src X
736 737 738 739 740
 * channel determines the result across all channels.  So to do a vec4
 * of this operation, we want to emit a scalar per source channel used
 * to produce dest channels.
 */
void
741
glsl_to_tgsi_visitor::emit_scalar(ir_instruction *ir, unsigned op,
742 743 744 745 746 747
        		        st_dst_reg dst,
        			st_src_reg orig_src0, st_src_reg orig_src1)
{
   int i, j;
   int done_mask = ~dst.writemask;

748
   /* TGSI RCP is a scalar operation splatting results to all channels,
749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785
    * like ARB_fp/vp.  So emit as many RCPs as necessary to cover our
    * dst channels.
    */
   for (i = 0; i < 4; i++) {
      GLuint this_mask = (1 << i);
      glsl_to_tgsi_instruction *inst;
      st_src_reg src0 = orig_src0;
      st_src_reg src1 = orig_src1;

      if (done_mask & this_mask)
         continue;

      GLuint src0_swiz = GET_SWZ(src0.swizzle, i);
      GLuint src1_swiz = GET_SWZ(src1.swizzle, i);
      for (j = i + 1; j < 4; j++) {
         /* If there is another enabled component in the destination that is
          * derived from the same inputs, generate its value on this pass as
          * well.
          */
         if (!(done_mask & (1 << j)) &&
             GET_SWZ(src0.swizzle, j) == src0_swiz &&
             GET_SWZ(src1.swizzle, j) == src1_swiz) {
            this_mask |= (1 << j);
         }
      }
      src0.swizzle = MAKE_SWIZZLE4(src0_swiz, src0_swiz,
        			   src0_swiz, src0_swiz);
      src1.swizzle = MAKE_SWIZZLE4(src1_swiz, src1_swiz,
        			  src1_swiz, src1_swiz);

      inst = emit(ir, op, dst, src0, src1);
      inst->dst.writemask = this_mask;
      done_mask |= this_mask;
   }
}

void
786
glsl_to_tgsi_visitor::emit_scalar(ir_instruction *ir, unsigned op,
787 788 789 790 791 792 793 794 795
        		        st_dst_reg dst, st_src_reg src0)
{
   st_src_reg undef = undef_src;

   undef.swizzle = SWIZZLE_XXXX;

   emit_scalar(ir, op, dst, src0, undef);
}

796 797 798 799
void
glsl_to_tgsi_visitor::emit_arl(ir_instruction *ir,
        		        st_dst_reg dst, st_src_reg src0)
{
800 801 802 803 804
   int op = TGSI_OPCODE_ARL;

   if (src0.type == GLSL_TYPE_INT || src0.type == GLSL_TYPE_UINT)
      op = TGSI_OPCODE_UARL;

805 806 807 808
   assert(dst.file == PROGRAM_ADDRESS);
   if (dst.index >= this->num_address_regs)
      this->num_address_regs = dst.index + 1;

809
   emit(NULL, op, dst, src0);
810 811
}

812
/**
813
 * Emit an TGSI_OPCODE_SCS instruction
814
 *
815 816 817 818
 * The \c SCS opcode functions a bit differently than the other TGSI opcodes.
 * Instead of splatting its result across all four components of the 
 * destination, it writes one value to the \c x component and another value to 
 * the \c y component.
819 820
 *
 * \param ir        IR instruction being processed
821 822
 * \param op        Either \c TGSI_OPCODE_SIN or \c TGSI_OPCODE_COS depending 
 *                  on which value is desired.
823 824 825 826
 * \param dst       Destination register
 * \param src       Source register
 */
void
827
glsl_to_tgsi_visitor::emit_scs(ir_instruction *ir, unsigned op,
828 829 830 831 832 833 834 835 836 837
        		     st_dst_reg dst,
        		     const st_src_reg &src)
{
   /* Vertex programs cannot use the SCS opcode.
    */
   if (this->prog->Target == GL_VERTEX_PROGRAM_ARB) {
      emit_scalar(ir, op, dst, src);
      return;
   }

838
   const unsigned component = (op == TGSI_OPCODE_SIN) ? 0 : 1;
839 840 841 842
   const unsigned scs_mask = (1U << component);
   int done_mask = ~dst.writemask;
   st_src_reg tmp;

843
   assert(op == TGSI_OPCODE_SIN || op == TGSI_OPCODE_COS);
844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885

   /* If there are compnents in the destination that differ from the component
    * that will be written by the SCS instrution, we'll need a temporary.
    */
   if (scs_mask != unsigned(dst.writemask)) {
      tmp = get_temp(glsl_type::vec4_type);
   }

   for (unsigned i = 0; i < 4; i++) {
      unsigned this_mask = (1U << i);
      st_src_reg src0 = src;

      if ((done_mask & this_mask) != 0)
         continue;

      /* The source swizzle specified which component of the source generates
       * sine / cosine for the current component in the destination.  The SCS
       * instruction requires that this value be swizzle to the X component.
       * Replace the current swizzle with a swizzle that puts the source in
       * the X component.
       */
      unsigned src0_swiz = GET_SWZ(src.swizzle, i);

      src0.swizzle = MAKE_SWIZZLE4(src0_swiz, src0_swiz,
        			   src0_swiz, src0_swiz);
      for (unsigned j = i + 1; j < 4; j++) {
         /* If there is another enabled component in the destination that is
          * derived from the same inputs, generate its value on this pass as
          * well.
          */
         if (!(done_mask & (1 << j)) &&
             GET_SWZ(src0.swizzle, j) == src0_swiz) {
            this_mask |= (1 << j);
         }
      }

      if (this_mask != scs_mask) {
         glsl_to_tgsi_instruction *inst;
         st_dst_reg tmp_dst = st_dst_reg(tmp);

         /* Emit the SCS instruction.
          */
886
         inst = emit(ir, TGSI_OPCODE_SCS, tmp_dst, src0);
887 888 889 890 891 892 893
         inst->dst.writemask = scs_mask;

         /* Move the result of the SCS instruction to the desired location in
          * the destination.
          */
         tmp.swizzle = MAKE_SWIZZLE4(component, component,
        			     component, component);
894
         inst = emit(ir, TGSI_OPCODE_SCS, dst, tmp);
895 896 897 898
         inst->dst.writemask = this_mask;
      } else {
         /* Emit the SCS instruction to write directly to the destination.
          */
899
         glsl_to_tgsi_instruction *inst = emit(ir, TGSI_OPCODE_SCS, dst, src0);
900 901 902 903 904 905 906
         inst->dst.writemask = scs_mask;
      }

      done_mask |= this_mask;
   }
}

907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922
int
glsl_to_tgsi_visitor::add_constant(gl_register_file file,
        		     gl_constant_value values[4], int size, int datatype,
        		     GLuint *swizzle_out)
{
   if (file == PROGRAM_CONSTANT) {
      return _mesa_add_typed_unnamed_constant(this->prog->Parameters, values,
                                              size, datatype, swizzle_out);
   } else {
      int index = 0;
      immediate_storage *entry;
      assert(file == PROGRAM_IMMEDIATE);

      /* Search immediate storage to see if we already have an identical
       * immediate that we can use instead of adding a duplicate entry.
       */
923 924
      foreach_list(node, &this->immediates) {
         entry = (immediate_storage *) node;
925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 940 941
         
         if (entry->size == size &&
             entry->type == datatype &&
             !memcmp(entry->values, values, size * sizeof(gl_constant_value))) {
             return index;
         }
         index++;
      }
      
      /* Add this immediate to the list. */
      entry = new(mem_ctx) immediate_storage(values, size, datatype);
      this->immediates.push_tail(entry);
      this->num_immediates++;
      return index;
   }
}

942
st_src_reg
943 944
glsl_to_tgsi_visitor::st_src_reg_for_float(float val)
{
945
   st_src_reg src(PROGRAM_IMMEDIATE, -1, GLSL_TYPE_FLOAT);
946
   union gl_constant_value uval;
947

948
   uval.f = val;
949
   src.index = add_constant(src.file, &uval, 1, GL_FLOAT, &src.swizzle);
950 951 952 953

   return src;
}

954
st_src_reg
955 956
glsl_to_tgsi_visitor::st_src_reg_for_int(int val)
{
957
   st_src_reg src(PROGRAM_IMMEDIATE, -1, GLSL_TYPE_INT);
958 959
   union gl_constant_value uval;
   
960
   assert(native_integers);
961 962

   uval.i = val;
963
   src.index = add_constant(src.file, &uval, 1, GL_INT, &src.swizzle);
964 965 966 967

   return src;
}

968
st_src_reg
969 970
glsl_to_tgsi_visitor::st_src_reg_for_type(int type, int val)
{
971
   if (native_integers)
972 973 974 975 976 977
      return type == GLSL_TYPE_FLOAT ? st_src_reg_for_float(val) : 
                                       st_src_reg_for_int(val);
   else
      return st_src_reg_for_float(val);
}

978 979 980 981 982 983 984 985 986 987 988 989 990 991 992 993 994 995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008
static int
type_size(const struct glsl_type *type)
{
   unsigned int i;
   int size;

   switch (type->base_type) {
   case GLSL_TYPE_UINT:
   case GLSL_TYPE_INT:
   case GLSL_TYPE_FLOAT:
   case GLSL_TYPE_BOOL:
      if (type->is_matrix()) {
         return type->matrix_columns;
      } else {
         /* Regardless of size of vector, it gets a vec4. This is bad
          * packing for things like floats, but otherwise arrays become a
          * mess.  Hopefully a later pass over the code can pack scalars
          * down if appropriate.
          */
         return 1;
      }
   case GLSL_TYPE_ARRAY:
      assert(type->length > 0);
      return type_size(type->fields.array) * type->length;
   case GLSL_TYPE_STRUCT:
      size = 0;
      for (i = 0; i < type->length; i++) {
         size += type_size(type->fields.structure[i].type);
      }
      return size;
   case GLSL_TYPE_SAMPLER:
1009
   case GLSL_TYPE_IMAGE:
1010 1011 1012 1013
      /* Samplers take up one slot in UNIFORMS[], but they're baked in
       * at link time.
       */
      return 1;
1014
   case GLSL_TYPE_ATOMIC_UINT:
1015
   case GLSL_TYPE_INTERFACE:
1016 1017 1018 1019
   case GLSL_TYPE_VOID:
   case GLSL_TYPE_ERROR:
      assert(!"Invalid type in type_size");
      break;
1020
   }
1021
   return 0;
1022 1023 1024 1025 1026
}

/**
 * In the initial pass of codegen, we assign temporary numbers to
 * intermediate results.  (not SSA -- variable assignments will reuse
1027
 * storage).
1028 1029 1030 1031 1032 1033
 */
st_src_reg
glsl_to_tgsi_visitor::get_temp(const glsl_type *type)
{
   st_src_reg src;

1034
   src.type = native_integers ? type->base_type : GLSL_TYPE_FLOAT;
1035
   src.reladdr = NULL;
1036 1037
   src.negate = 0;

1038 1039 1040
   if (!options->EmitNoIndirectTemp &&
       (type->is_array() || type->is_matrix())) {

1041 1042 1043 1044 1045 1046 1047 1048 1049 1050
      src.file = PROGRAM_ARRAY;
      src.index = next_array << 16 | 0x8000;
      array_sizes[next_array] = type_size(type);
      ++next_array;

   } else {
      src.file = PROGRAM_TEMPORARY;
      src.index = next_temp;
      next_temp += type_size(type);
   }
1051 1052 1053 1054

   if (type->is_array() || type->is_record()) {
      src.swizzle = SWIZZLE_NOOP;
   } else {
1055
      src.swizzle = swizzle_for_size(type->vector_elements);
1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066
   }

   return src;
}

variable_storage *
glsl_to_tgsi_visitor::find_variable_storage(ir_variable *var)
{
   
   variable_storage *entry;

1067 1068
   foreach_list(node, &this->variables) {
      entry = (variable_storage *) node;
1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082

      if (entry->var == var)
         return entry;
   }

   return NULL;
}

void
glsl_to_tgsi_visitor::visit(ir_variable *ir)
{
   if (strcmp(ir->name, "gl_FragCoord") == 0) {
      struct gl_fragment_program *fp = (struct gl_fragment_program *)this->prog;

1083 1084
      fp->OriginUpperLeft = ir->data.origin_upper_left;
      fp->PixelCenterInteger = ir->data.pixel_center_integer;
1085 1086
   }

1087
   if (ir->data.mode == ir_var_uniform && strncmp(ir->name, "gl_", 3) == 0) {
1088 1089 1090 1091 1092 1093 1094 1095 1096 1097 1098 1099 1100 1101 1102 1103
      unsigned int i;
      const ir_state_slot *const slots = ir->state_slots;
      assert(ir->state_slots != NULL);

      /* Check if this statevar's setup in the STATE file exactly
       * matches how we'll want to reference it as a
       * struct/array/whatever.  If not, then we need to move it into
       * temporary storage and hope that it'll get copy-propagated
       * out.
       */
      for (i = 0; i < ir->num_state_slots; i++) {
         if (slots[i].swizzle != SWIZZLE_XYZW) {
            break;
         }
      }

1104
      variable_storage *storage;
1105 1106 1107 1108 1109 1110 1111 1112 1113 1114 1115 1116 1117 1118
      st_dst_reg dst;
      if (i == ir->num_state_slots) {
         /* We'll set the index later. */
         storage = new(mem_ctx) variable_storage(ir, PROGRAM_STATE_VAR, -1);
         this->variables.push_tail(storage);

         dst = undef_dst;
      } else {
         /* The variable_storage constructor allocates slots based on the size
          * of the type.  However, this had better match the number of state
          * elements that we're going to copy into the new temporary.
          */
         assert((int) ir->num_state_slots == type_size(ir->type));

1119 1120 1121
         dst = st_dst_reg(get_temp(ir->type));

         storage = new(mem_ctx) variable_storage(ir, dst.file, dst.index);
1122

1123
         this->variables.push_tail(storage);
1124 1125 1126 1127 1128 1129 1130 1131 1132 1133 1134 1135 1136 1137
      }


      for (unsigned int i = 0; i < ir->num_state_slots; i++) {
         int index = _mesa_add_state_reference(this->prog->Parameters,
        				       (gl_state_index *)slots[i].tokens);

         if (storage->file == PROGRAM_STATE_VAR) {
            if (storage->index == -1) {
               storage->index = index;
            } else {
               assert(index == storage->index + (int)i);
            }
         } else {
1138 1139 1140 1141 1142 1143
         	/* We use GLSL_TYPE_FLOAT here regardless of the actual type of
         	 * the data being moved since MOV does not care about the type of
         	 * data it is moving, and we don't want to declare registers with
         	 * array or struct types.
         	 */
            st_src_reg src(PROGRAM_STATE_VAR, index, GLSL_TYPE_FLOAT);
1144
            src.swizzle = slots[i].swizzle;
1145
            emit(ir, TGSI_OPCODE_MOV, dst, src);
1146 1147 1148 1149 1150 1151 1152 1153 1154 1155 1156 1157 1158 1159 1160 1161 1162 1163
            /* even a float takes up a whole vec4 reg in a struct/array. */
            dst.index++;
         }
      }

      if (storage->file == PROGRAM_TEMPORARY &&
          dst.index != storage->index + (int) ir->num_state_slots) {
         fail_link(this->shader_program,
        	   "failed to load builtin uniform `%s'  (%d/%d regs loaded)\n",
        	   ir->name, dst.index - storage->index,
        	   type_size(ir->type));
      }
   }
}

void
glsl_to_tgsi_visitor::visit(ir_loop *ir)
{
1164
   emit(NULL, TGSI_OPCODE_BGNLOOP);
1165 1166 1167

   visit_exec_list(&ir->body_instructions, this);

1168
   emit(NULL, TGSI_OPCODE_ENDLOOP);
1169 1170 1171 1172 1173 1174 1175
}

void
glsl_to_tgsi_visitor::visit(ir_loop_jump *ir)
{
   switch (ir->mode) {
   case ir_loop_jump::jump_break:
1176
      emit(NULL, TGSI_OPCODE_BRK);
1177 1178
      break;
   case ir_loop_jump::jump_continue:
1179
      emit(NULL, TGSI_OPCODE_CONT);
1180 1181 1182 1183 1184 1185 1186 1187 1188 1189 1190 1191 1192 1193 1194 1195 1196 1197 1198 1199 1200 1201
      break;
   }
}


void
glsl_to_tgsi_visitor::visit(ir_function_signature *ir)
{
   assert(0);
   (void)ir;
}

void
glsl_to_tgsi_visitor::visit(ir_function *ir)
{
   /* Ignore function bodies other than main() -- we shouldn't see calls to
    * them since they should all be inlined before we get to glsl_to_tgsi.
    */
   if (strcmp(ir->name, "main") == 0) {
      const ir_function_signature *sig;
      exec_list empty;

1202
      sig = ir->matching_signature(NULL, &empty);
1203 1204 1205

      assert(sig);

1206 1207
      foreach_list(node, &sig->body) {
         ir_instruction *ir = (ir_instruction *) node;
1208 1209 1210 1211 1212 1213

         ir->accept(this);
      }
   }
}

1214
bool
1215 1216 1217 1218
glsl_to_tgsi_visitor::try_emit_mad(ir_expression *ir, int mul_operand)
{
   int nonmul_operand = 1 - mul_operand;
   st_src_reg a, b, c;
1219
   st_dst_reg result_dst;
1220 1221 1222 1223 1224 1225 1226 1227 1228 1229 1230 1231 1232

   ir_expression *expr = ir->operands[mul_operand]->as_expression();
   if (!expr || expr->operation != ir_binop_mul)
      return false;

   expr->operands[0]->accept(this);
   a = this->result;
   expr->operands[1]->accept(this);
   b = this->result;
   ir->operands[nonmul_operand]->accept(this);
   c = this->result;

   this->result = get_temp(ir->type);
1233 1234 1235
   result_dst = st_dst_reg(this->result);
   result_dst.writemask = (1 << ir->type->vector_elements) - 1;
   emit(ir, TGSI_OPCODE_MAD, result_dst, a, b, c);
1236 1237 1238 1239

   return true;
}

1240 1241 1242 1243 1244 1245 1246 1247 1248 1249 1250 1251 1252 1253 1254 1255 1256 1257 1258 1259 1260 1261 1262 1263 1264 1265 1266 1267 1268 1269 1270 1271 1272 1273 1274 1275 1276 1277 1278 1279
/**
 * Emit MAD(a, -b, a) instead of AND(a, NOT(b))
 *
 * The logic values are 1.0 for true and 0.0 for false.  Logical-and is
 * implemented using multiplication, and logical-or is implemented using
 * addition.  Logical-not can be implemented as (true - x), or (1.0 - x).
 * As result, the logical expression (a & !b) can be rewritten as:
 *
 *     - a * !b
 *     - a * (1 - b)
 *     - (a * 1) - (a * b)
 *     - a + -(a * b)
 *     - a + (a * -b)
 *
 * This final expression can be implemented as a single MAD(a, -b, a)
 * instruction.
 */
bool
glsl_to_tgsi_visitor::try_emit_mad_for_and_not(ir_expression *ir, int try_operand)
{
   const int other_operand = 1 - try_operand;
   st_src_reg a, b;

   ir_expression *expr = ir->operands[try_operand]->as_expression();
   if (!expr || expr->operation != ir_unop_logic_not)
      return false;

   ir->operands[other_operand]->accept(this);
   a = this->result;
   expr->operands[0]->accept(this);
   b = this->result;

   b.negate = ~b.negate;

   this->result = get_temp(ir->type);
   emit(ir, TGSI_OPCODE_MAD, st_dst_reg(this->result), a, b, a);

   return true;
}

1280
bool
1281 1282
glsl_to_tgsi_visitor::try_emit_sat(ir_expression *ir)
{
1283
   /* Emit saturates in the vertex shader only if SM 3.0 is supported.
1284
    */
1285 1286
   if (this->prog->Target == GL_VERTEX_PROGRAM_ARB &&
       !st_context(this->ctx)->has_shader_model3) {
1287
      return false;
1288
   }
1289 1290 1291 1292 1293 1294 1295 1296

   ir_rvalue *sat_src = ir->as_rvalue_to_saturate();
   if (!sat_src)
      return false;

   sat_src->accept(this);
   st_src_reg src = this->result;

1297 1298 1299 1300 1301 1302 1303 1304 1305 1306 1307 1308 1309 1310 1311 1312 1313 1314 1315 1316 1317 1318 1319 1320 1321 1322
   /* If we generated an expression instruction into a temporary in
    * processing the saturate's operand, apply the saturate to that
    * instruction.  Otherwise, generate a MOV to do the saturate.
    *
    * Note that we have to be careful to only do this optimization if
    * the instruction in question was what generated src->result.  For
    * example, ir_dereference_array might generate a MUL instruction
    * to create the reladdr, and return us a src reg using that
    * reladdr.  That MUL result is not the value we're trying to
    * saturate.
    */
   ir_expression *sat_src_expr = sat_src->as_expression();
   if (sat_src_expr && (sat_src_expr->operation == ir_binop_mul ||
			sat_src_expr->operation == ir_binop_add ||
			sat_src_expr->operation == ir_binop_dot)) {
      glsl_to_tgsi_instruction *new_inst;
      new_inst = (glsl_to_tgsi_instruction *)this->instructions.get_tail();
      new_inst->saturate = true;
   } else {
      this->result = get_temp(ir->type);
      st_dst_reg result_dst = st_dst_reg(this->result);
      result_dst.writemask = (1 << ir->type->vector_elements) - 1;
      glsl_to_tgsi_instruction *inst;
      inst = emit(ir, TGSI_OPCODE_MOV, result_dst, src);
      inst->saturate = true;
   }
1323 1324 1325 1326 1327 1328 1329 1330

   return true;
}

void
glsl_to_tgsi_visitor::reladdr_to_temp(ir_instruction *ir,
        			    st_src_reg *reg, int *num_reladdr)
{
1331
   if (!reg->reladdr && !reg->reladdr2)
1332 1333
      return;

1334 1335
   if (reg->reladdr) emit_arl(ir, address_reg, *reg->reladdr);
   if (reg->reladdr2) emit_arl(ir, address_reg2, *reg->reladdr2);
1336 1337 1338 1339

   if (*num_reladdr != 1) {
      st_src_reg temp = get_temp(glsl_type::vec4_type);

1340
      emit(ir, TGSI_OPCODE_MOV, st_dst_reg(temp), *reg);
1341 1342 1343 1344 1345 1346 1347 1348 1349 1350 1351 1352 1353 1354
      *reg = temp;
   }

   (*num_reladdr)--;
}

void
glsl_to_tgsi_visitor::visit(ir_expression *ir)
{
   unsigned int operand;
   st_src_reg op[Elements(ir->operands)];
   st_src_reg result_src;
   st_dst_reg result_dst;

1355
   /* Quick peephole: Emit MAD(a, b, c) instead of ADD(MUL(a, b), c)
1356 1357 1358 1359 1360 1361 1362
    */
   if (ir->operation == ir_binop_add) {
      if (try_emit_mad(ir, 1))
         return;
      if (try_emit_mad(ir, 0))
         return;
   }
1363 1364 1365 1366 1367 1368 1369 1370 1371 1372

   /* Quick peephole: Emit OPCODE_MAD(-a, -b, a) instead of AND(a, NOT(b))
    */
   if (ir->operation == ir_binop_logic_and) {
      if (try_emit_mad_for_and_not(ir, 1))
	 return;
      if (try_emit_mad_for_and_not(ir, 0))
	 return;
   }

1373 1374 1375
   if (try_emit_sat(ir))
      return;

1376 1377
   if (ir->operation == ir_quadop_vector)
      assert(!"ir_quadop_vector should have been lowered");
1378 1379 1380 1381 1382 1383

   for (operand = 0; operand < ir->get_num_operands(); operand++) {
      this->result.file = PROGRAM_UNDEFINED;
      ir->operands[operand]->accept(this);
      if (this->result.file == PROGRAM_UNDEFINED) {
         printf("Failed to get tree for expression operand:\n");
1384 1385
         ir->operands[operand]->print();
         printf("\n");
1386 1387 1388 1389 1390 1391 1392 1393 1394 1395 1396 1397 1398 1399 1400 1401 1402 1403 1404 1405 1406 1407 1408 1409 1410 1411 1412 1413 1414 1415 1416 1417
         exit(1);
      }
      op[operand] = this->result;

      /* Matrix expression operands should have been broken down to vector
       * operations already.
       */
      assert(!ir->operands[operand]->type->is_matrix());
   }

   int vector_elements = ir->operands[0]->type->vector_elements;
   if (ir->operands[1]) {
      vector_elements = MAX2(vector_elements,
        		     ir->operands[1]->type->vector_elements);
   }

   this->result.file = PROGRAM_UNDEFINED;

   /* Storage for our result.  Ideally for an assignment we'd be using
    * the actual storage for the result here, instead.
    */
   result_src = get_temp(ir->type);
   /* convenience for the emit functions below. */
   result_dst = st_dst_reg(result_src);
   /* Limit writes to the channels that will be used by result_src later.
    * This does limit this temp's use as a temporary for multi-instruction
    * sequences.
    */
   result_dst.writemask = (1 << ir->type->vector_elements) - 1;

   switch (ir->operation) {
   case ir_unop_logic_not:
1418
      if (result_dst.type != GLSL_TYPE_FLOAT)
1419
         emit(ir, TGSI_OPCODE_NOT, result_dst, op[0]);
1420 1421 1422 1423 1424 1425 1426 1427 1428
      else {
         /* Previously 'SEQ dst, src, 0.0' was used for this.  However, many
          * older GPUs implement SEQ using multiple instructions (i915 uses two
          * SGE instructions and a MUL instruction).  Since our logic values are
          * 0.0 and 1.0, 1-x also implements !x.
          */
         op[0].negate = ~op[0].negate;
         emit(ir, TGSI_OPCODE_ADD, result_dst, op[0], st_src_reg_for_float(1.0));
      }
1429 1430
      break;
   case ir_unop_neg:
1431
      if (result_dst.type == GLSL_TYPE_INT || result_dst.type == GLSL_TYPE_UINT)
1432 1433 1434 1435 1436
         emit(ir, TGSI_OPCODE_INEG, result_dst, op[0]);
      else {
         op[0].negate = ~op[0].negate;
         result_src = op[0];
      }
1437 1438
      break;
   case ir_unop_abs:
1439
      emit(ir, TGSI_OPCODE_ABS, result_dst, op[0]);
1440 1441
      break;
   case ir_unop_sign:
1442
      emit(ir, TGSI_OPCODE_SSG, result_dst, op[0]);
1443 1444
      break;
   case ir_unop_rcp:
1445
      emit_scalar(ir, TGSI_OPCODE_RCP, result_dst, op[0]);
1446 1447 1448
      break;

   case ir_unop_exp2:
1449
      emit_scalar(ir, TGSI_OPCODE_EX2, result_dst, op[0]);
1450 1451 1452 1453 1454 1455
      break;
   case ir_unop_exp:
   case ir_unop_log:
      assert(!"not reached: should be handled by ir_explog_to_explog2");
      break;
   case ir_unop_log2:
1456
      emit_scalar(ir, TGSI_OPCODE_LG2, result_dst, op[0]);
1457 1458
      break;
   case ir_unop_sin:
1459
      emit_scalar(ir, TGSI_OPCODE_SIN, result_dst, op[0]);
1460 1461
      break;
   case ir_unop_cos:
1462
      emit_scalar(ir, TGSI_OPCODE_COS, result_dst, op[0]);
1463 1464
      break;
   case ir_unop_sin_reduced:
1465
      emit_scs(ir, TGSI_OPCODE_SIN, result_dst, op[0]);