There are plenty of comments in the code, but if anyone has questions or problems please comment.

require 'find'

class Hash
  # sets value at a dot seperated key path
  # key = 'general.name' = hash[:general][:name]
  def set_value_for_key(key, value)
    key_parts = key.split('.')
    if key_parts.length > 1
      key_part = key_parts[0]
      new_key = key_parts[1..-1].join('.')
      self[key_part] ||= {}
      self[key_part].set_value_for_key(new_key, value)
    else
      self[key] = value
    end
  end
end

module I18n
  def self.translation_exception(*args)
    locale = args[1] # => :en
    key = args[2] # => 'general.name'
    #load existing locale data
    locale_file = Rails.root.join("config/locales/#{locale}.yml")
    data = YAML.load_file(locale_file) if File.exists?(locale_file)
    data ||= {}
    # normalise the key
    normalized_key = I18n.normalize_keys(locale, key, []) # => [:general,:name]
    # used for the value if
    value_text = normalized_key[-1] # => :name
    # add to the hash
    data.set_value_for_key(normalized_key.join('.'), value_text.to_s.humanize)
    # save
    File.open(locale_file, "w") {|f| f.write( data.to_yaml) }
  end
end
# override default exception for translation exceptions
# this allows us to process missing translation with the above code
I18n.exception_handler = :translation_exception

namespace :translations do
  desc "Update translation yaml from views"
  task :from_views => :environment do
    assets = []
    #get all the view files
    Find.find(Rails.configuration.view_path) do |view|
      if File.file?(view) && File.extname(view) == '.erb'
        File.readlines(view).each do |line|
          #look for t or I18n.t calls on each line, and append to assets
          assets += line.scan(/\st\(([^,\)]*)/).flatten
          assets += line.scan(/\sI18n.t\(([^,\)]*)/).flatten
        end
      end
    end
    #clean the keys up
    assets.map!{|a| a.gsub("'","").gsub(':','')}
    #sort so deep keys are last
    assets.sort!{|x,y| x.scan('.').count <=> y.scan('.').count}
    #for each locale
    I18n.available_locales.each do |locale|
      I18n.locale = locale
      #attempt to call translate for each key
      assets.each do |translation_key|
        I18n.t(translation_key)
      end
    end

  end

  desc "Update translation yaml from models"
  task :from_models => :environment do
    #require the model files so rails is aware of the model
    Find.find(Rails.root.join('app/models')) do |model_file|
      if File.file?(model_file) && File.extname(model_file) == '.rb'
        require model_file
      end
    end
    #for each available locale
    I18n.available_locales.each do |locale|
      I18n.locale = locale
      #and each available model
      ActiveRecord::Base.descendants.each do |model|
        #in my case some models are for extension only and will not have a table
        #so using a rescue block lets the code keep on going
        begin
          model_name = model.name.underscore
          #for each column, try to access the translation
          model.columns.each do |col|
            I18n.t("activerecord.attributes.#{model_name}.#{col.name}".trim)
          end
        rescue Exception => ex
          #do nothing
        end
      end
    end
  end

  desc "Update translation yaml from all sources"
  task :from_all => [:from_views, :from_models]

end

Setup

You will need Ruby (I was using 1.9.2-p180 via RVM) the googlecalendar gem the mail gem and optionally nokogiri if you want to parse a multipart html message.

You can install all with: gem install googlecalendar mail nokogiri

You will need to add to your /etc/aliases file (for postfix) for other MTA’s please RTFM:

scriptaddress: “| /path/to/script.rb”

Where script address is the address where the message should be emailed to.

The skeleton script is below:

#!/usr/local/bin/ruby
require 'rubygems'
require 'nokogiri'
require 'googlecalendar'
require 'time'
require 'mail'

include Googlecalendar

#read the email in
email = Mail.new(STDIN.read)

#if it's a multipart message then email.parts[1].body.to_s will give me the HTML
#I can then use Nokogiri to give me a HTML doc
#html_doc = Nokogiri::HTML(email.parts[1].body.to_s)

g = GData.new
#logon to my account and fetch a token
token = g.login('danny.hawkins@gmail.com', 'yourpassword')

event = {
  :title      => email.subject,
  :content    => email.body,
  :author     => "dannyhawkins",
  :email      => "danny.hawkins@gmail",
  :where      => "London",
  :startTime  => "2011-03-10T15:00:00.000+00:00", #+00:00 means GMT +01:00 would be CET etc
  :endTime    => "2011-03-10T16:00:00.000+00:00"
}
#post my event
response = g.new_event(event)
#post my event to a different calendar
#response = g.new_event(event, 'calendar name')

require_once('PHPUnit/Util/Log/JUnit.php');

function bfglob($path, $pattern = '*', $flags = 0, $depth = 0) {
  $matches = array();
  $folders = array(rtrim($path, DIRECTORY_SEPARATOR));

  while($folder = array_shift($folders)) {
    $matches = array_merge($matches, glob($folder.DIRECTORY_SEPARATOR.$pattern, $flags));
    if($depth != 0) {
      $moreFolders = glob($folder.DIRECTORY_SEPARATOR.'*', GLOB_ONLYDIR);
      $depth   = ($depth < -1) ? -1: $depth + count($moreFolders) - 2;
      $folders = array_merge($folders, $moreFolders);
    }
  }
  return $matches;
}

class SPTestRunner{
  public static function run(){
    // Create the test suite instance
    $suite = new PHPUnit_Framework_TestSuite();
    $suite->setName('SPTestRunner');

    $filenames = bfglob(__DIR__,"*Test.php",0,-1);
    // Add files to the TestSuite
    $suite->addTestFiles($filenames);

    // Create a xml listener object
    $listener = new PHPUnit_Util_Log_JUnit;

    // Create TestResult object and pass the xml listener to it
    $testResult = new PHPUnit_Framework_TestResult();
    $testResult->addListener($listener);

    // Run the TestSuite
    $result = $suite->run($testResult);

    // Get the results from the listener
    $xml_result = $listener->getXML();
    return $xml_result;
  }
}

$test_xml = SPTestRunner::run();
$simple = new SimpleXMLElement($test_xml);

$test_results = array();
$assertions = 0;
$time = 0.0;
$tests = 0;
foreach($simple->{'testsuite'}->testsuite as $testsuite){
  foreach($testsuite->testcase as $testcase){
    $result = array();
    $tests += 1;
    // Don't froget to cast SimpleXMLElement to string!
    $result['name'] = (string)$testcase['name'];
    $result['suite'] = (string)$testcase['class'];
    $assertions += $result['assertions'] = (string)$testcase['assertions'];
    $time += $result['time'] = (string)$testcase['time'];
    if(isset($testcase->{'failure'})){
      $result['result'] = 'Fail';
      $result['message'] = (string)$testcase->{'failure'};
    }else{
      $result['result'] = 'Pass';
      $result['message'] = '';
    }
    $test_results[] = $result;
  }
}
?>




Suite
Test Name
Result
Message
Assertions
Time



  




      
Fail

       
Pass

      





  


 TESTS
 ASSERTIONS
 SEC

It’s by no means pretty, but it does the job, just drop it in a folder at the root of your tests and it will look for all files including subdirectories that look like *Test.php and run them. Unfortunately because the script first has to wait for the XML to come back form PHPUnit you will not see the result of your tests until they have all finished running.

AJAX PHPUnit Runner

I’m hoping to find some time to able to realise an AJAX driven interface that allow you to select certain groups / tests / suites and run them in browser with realtime results. If anyway wants to lend me a hand I’m open to offers.

The syntax is pretty simple:

      checkout_command = GoogleCheckout::Order.frontend.create_checkout_command
      checkout_command.rounding_rule = Google4R::Checkout::RoundingRule::PER_LINE
      checkout_command.rounding_mode = Google4R::Checkout::RoundingMode::HALF_UP

The problem I had was that I need to make sure notices are shown on ajax requests as well as normal requests. I started out by having a block of code at the bottom of each js.erb that would show notices like so:

%('#notices').replaceWith('<% escape_javascript(render :partial => "shared/notices")%>');

Which was all well and good except it’s not DRY, and didn’t feel very rails at all. My final solution involved using application.js.erb in layouts, which I didn’t even realise was possible. Check it out:

#application.js.erb

<%= render :partial => "shared/notices" %>

<%= yield %>

#shared/notices.js.erb

if($('#notices')){
  $('#notices').replaceWith('<% escape_javascript(render :partial => "shared/notices.html")%>');
}else{
  notices = $('#header').before('<% escape_javascript(render :partial => "shared/notices.html")%>');
}

#shared/notices.html.erb

  <% if flash[:notice] %>
<%= flash[:notice] %>

  &lt% end %>

  <% if flash[:error] %>
>%= flash[:error] %>

  <% end %%gt;

  <%= link_to "close", "#", :class => "close" %>
  <% javascript_tag do %>
    noticesTimer = setTimeout(function(){ $('#notices').slideUp()}, 5000);
    $('#notices .close').click(function(){$('#notices').slideUp(); return false; });
  <% end %>

It will only load notices if needed, replace any #notices if it already exists and self hide afterwards.

This has always been a long term goal of thebatteryguys.co.uk, who started of by spending a considerable amount on CPC advertising in order to allow people to be made aware of their top class service. Their core business is next day delivery of batteries for the automative industry. I did not anticipate that in such a short period of time they would have gained this much momentum and achieved the number 1 spot, the optimisations we have made to the site have contributed towards the result, but it’s also plain to see that the amazing service these guys offer has been talked about and people have come to the site through recommendations and not just advertising. Well done guys.

Acts as state machine (AASM) is a rubygem for adding finite state machines to Ruby classes. http://github.com/rubyist/aasm. This is a widely used gem/plugin and is the de-facto standard for state management in Ruby on Rails.

Whilst working on the Technophobia Video Uploader (coming soon!!!) I hit a problem with aasm. I’ve used the gem before and think it can be overkill for certain situations but when used correctly has massive benefits.

How it works

The way acts as state machine works is by defining various states and event. The events have rules that determine which state transitions can take place and callbacks which allow functions to be executed at various stages of the transition. Take the code below for example:

 # State Machine
  aasm_column :status
  aasm_initial_state :pending

  aasm_state :pending
  aasm_state :encoded, :enter => :encode_video
  aasm_state :uploaded, :enter => :upload_video

  aasm_state :encode_failed
  aasm_state :upload_failed
  aasm_state :complete

  # State Machine Transitions
  aasm_event :encode do
    transitions :from => :pending, :to => :encoded
  end

  aasm_event :upload do
    transitions :from => :encoded, :to => :uploaded
  end

  aasm_event :complete do
    transitions :from => :uploaded, :to => :complete
  end

  aasm_event :failed do
    transitions :from => :pending, :to => :encode_failed
    transitions :from => :encoded, :to => :upload_failed
  end

This gives me 4 events that will cause the Objects state to move through 6 defined states. By calling Object.encode! I fire the encode event which will transition from the pending state to the encoded state going through any callbacks on the way. The callback method “encode_video” defined for the encode event is fired on entry into the transition. If the object state is anything other than pending I will get and InvalidStateTransition error as the encode! event should only allow pending objects to be processed.

The problem

The sequence which the callbacks and state change is called can be problematic, in the example above I use the enter callback to execute a method “encode_video”, this callback fires before the event status is changed.

    #callbacks and state change
    state.call_action(:before_enter, self)
    state.call_action(:enter, self) #our callback is executed
    self.aasm_current_state = state_name #the state is changed
    state.call_action(:after_enter, self)

This which works great unless you need to change state within the callback method, for example if the callback fails and you change the state to failed.
In my encode_video method I check to see if the process has been successful and if not I call the failed! event, this works and sets the status to the appropriate failure status. However when control is returned to the block of code above, the line “self.aasm_current_state = state_name” changes the state from my failure state to the state as defined in the transition.

The Solution

Fortunately there is a very simple solution do this. You can pass the :error option into the aasm_event method to set a callback that will be used when a StandardError is raised within one of the state callbacks. This will allow the program to keep running but will break out of the code at the correct point before the state is changed. So we change the state machine transitions like so:

  # State Machine Transitions
  aasm_event :encode, :error => :error_raised do
    transitions :from => :pending, :to => :encoded
  end

  aasm_event :upload, :error => :error_raised do
    transitions :from => :encoded, :to => :uploaded
  end

  aasm_event :complete, :error => :error_raised do
    transitions :from => :uploaded, :to => :complete
  end

Then define the error method:

  def error_raised(error)
    failed!
    logger.debug("Error raised in video_encoding #{error.message}")
  end

And finally raise the correct error class in the call back function:

  def encode_video
    begin
      encoder = VideoChimp::Encoder.new
      case encoding_profile.container
        when "mp4" then
          encoder.encode_mp4
        when "flv" then
          encoder.encode_flv
      end
      unless encoder.valid?
        raise StandardError.new("Something went wrong encoding")
      end
    rescue Exception => ex
      raise StandardError.new("Exception in encode_video #{ex.message}")
    end
  end

And thats it. More control over your state machine

//unobtrusive javascript replacements
//setup for format.js block
$.ajaxSetup({
	'beforeSend': function(xhr) { xhr.setRequestHeader('Accept', 'text/javascript') }
});

//for replacing remote_form_form
$.fn.submitWithAjax = function(){
	this.live('submit',function(){
		$.ajax({
			url: $(this).attr("action"),
			type: ($(this.form).find("[name=_method]").attr('value') == undefined) ? 'post' : $(this.form).find("[name=_method]").attr('value'),
			data: $(this).serialize(),
			dataType: 'script'
		});
		return false;
	});
}

//for replacing link_to_remote
$.fn.linkWithAjax = function(){
	//add new click binding
	this.live('click',function(){
		$.ajax({
			url: this.href,
			type: $(this).is('.delete') ? 'delete' : 'get',
			dataType: 'script'
		});
		return false;
	});
}

//Bind links and forms on load
$(function(){
	$('a.remote').linkWithAjax();
	$('form.remote').submitWithAjax();
});

Then include jquery 1.4.2 min and ajax.js in your header, my javascript include line looks like this:

  javascript_include_tag 'jquery','jquery-ui','jrails','ajax', :cache => true

Then all you have to do is give the form_for or link_to tags a class of remote, if you want to delete make sure you give the link a class of delete too. Please note the ajax.js file I have created expects a script back, but you can pass expect html or text content by removing dataType: ‘script’

If you do remove the dataType: ‘script’ option you may want to add a success function. so…

     this.live('click',function(){
		$.ajax({
			url: this.href,
			type: $(this).is('.delete') ? 'delete' : 'get',
			success: function(data){
                             alert(data);
                        }
		});
		return false;
	});

Hope it helps, post a comment if you have a question.

De-normalisation is just one of many strategies to cope with the scaling requirements of an application. It is the process of devising a method to group relevant data together in an optimal way for the system to return what is required quickly.

During the talk Daniel and Matt explained how Songkick has devised a series of classes to implement a de-normalisation strategy using a key value store in a document based database (MongoDB).

They use the Model View Presenter (MVP Presenter first) design pattern to serve up content for the views on the website. Presenters are first class domain objects tailored for displaying a single page or module of information. So when looking at information on the site you are actually seeing content returned from a document store through the Presenter.

This however presents it’s own bunch of problems. The first and probably most obvious is consistency. Unlike caching, content is not expired and then re-generated on demand, content will be regenerated as and when it is created or updated.

Presenter, Soli and Silovator

A presenter is basically a model which has methods to get only the data required for the view element and store these in the specific document. A typical presenter looks like this:

class EventListingPresenter

  def initialize(event)
    @event = event
  end

  def title;            @event.artists.map(&:name).to_sentance;   end
  def image_count;      @event.images.count;                      end
  def attendance_count; @event.attending_users.count;             end

  #silo creates document on save using silo methods

  silo_method :title
  silo_method :image_count
  silo_method :attendance_count

ends

The silo_method works in a similar way to memoization. Data will attempt to return from the document store and failover to using the actual database through ActiveRecord.

Cache control

Cache expiration is controlled through asynchronous observers called Silovators. These silovators listen for events like create, update and delete on objects and run processes to generate the presenter data for any given presenter. For example:

class EventListingSilovator < AsyncObserver

  listen :create, Attendance do
    Silo.generate(presenter_for(attendance.event))
  end

  listen :update, Venue do
    venue.events.each do |event|
      Silo.generate(presenter_for(event.venue))
    end
  end
end

The silovator processes could be running on any number of machines, which creates another problem with possible race conditions. This is solved using http://github.com/songkick/mega_mutex which will ensure when a job is picked up, no other client will run that job at the same time. Megamutex uses memcache-client so your environment must be setup to use it.

Songkick have multiple background tasks which consume events and perform actions, these include ImageResizer, EmailNotifier and Silovater which all take advantage of mega_mutex to guarantee consistency.

Resources

• http://www.atomicobject.com/files/PF_March2005.pdf
• http://github.com/songkick/mega_mutex