From a908f00771899082e852cc81ef21bce1c8250aca Mon Sep 17 00:00:00 2001 From: Michael Brown Date: Sat, 15 Jun 2013 01:25:41 -0400 Subject: [PATCH 1/4] Documentation update: work in progress --- config/clock.rb | 9 +++++ config/discourse.pill.sample | 23 ++++++++--- config/locales/server.en.yml | 2 +- config/logrotate.conf | 38 ++++++++++++++++++ config/nginx.sample.conf | 7 ++++ docs/INSTALL-ubuntu.md | 16 ++++---- docs/TROUBLESHOOTING-prod.md | 77 ++++++++++++++++++++++++++++++++++++ 7 files changed, 158 insertions(+), 14 deletions(-) create mode 100644 config/logrotate.conf create mode 100644 docs/TROUBLESHOOTING-prod.md diff --git a/config/clock.rb b/config/clock.rb index bc0c4962e..7183123f5 100644 --- a/config/clock.rb +++ b/config/clock.rb @@ -5,6 +5,15 @@ require_relative 'environment' # These are jobs you should run on a regular basis to make your # forum work properly. +def setup_log + Clockwork.configure do |config| + config[:logger] = Logger.new(ENV["CLOCK_LOG"]) + end if ENV["CLOCK_LOG"] +end + +trap('HUP') { setup_log } +setup_log + module Clockwork handler do |job| Jobs.enqueue(job, all_sites: true) diff --git a/config/discourse.pill.sample b/config/discourse.pill.sample index 592da646a..a9afb4817 100644 --- a/config/discourse.pill.sample +++ b/config/discourse.pill.sample @@ -20,11 +20,22 @@ Bluepill.application("discourse", :base_dir => ENV["HOME"] + '/.bluepill') do |a # getting this to work was a nightmare # bundle exec spawns a process totally messing with the demonize option # so we suck the environment out and set it up first - app.environment = `env -i BUNDLE_GEMFILE=#{rails_root}/Gemfile /usr/local/rvm/bin/bootup_bundle exec env`.lines.inject({}) do |env_hash,l| + bootup_bundle = [ "#{ENV['HOME']}/.rvm/bin/rvm/bootup_bundle", + "/usr/local/rvm/bin/rvm/bootup_bundle", + `which bootup_bundle`.strip, + ].each do |location| + if File.exist? location + break location + end + end + # XXX if none match, bootup_bundle is set to the array + + if bootup_bundle + app.environment = `env -i BUNDLE_GEMFILE=#{rails_root}/Gemfile #{bootup_bundle} exec env`.lines.inject({}) do |env_hash,l| kv = l.chomp.split('=',2) env_hash[kv[0]] = kv[1] env_hash - end if File.exist?("/usr/local/rvm/bin/rvm") + end app.environment ||= {} app.environment['RAILS_ENV'] = rails_env @@ -37,7 +48,7 @@ Bluepill.application("discourse", :base_dir => ENV["HOME"] + '/.bluepill') do |a File.directory? sockdir or Dir.mkdir sockdir num_webs.times do |i| app.process("thin-#{i}") do |process| - process.start_command = "bundle exec thin start -e production -t 0 --socket #{sockdir}/thin.#{i}.sock -P #{rails_root}/tmp/pids/thin#{i}.pid -d" + process.start_command = "bundle exec thin start -e production -t 0 --socket #{sockdir}/thin.#{i}.sock --pid #{rails_root}/tmp/pids/thin#{i}.pid --log #{rails_root}/log/thin-#{i}.log --daemonize" # Alternatively, you can start with a port number instead of a socket. If you do that, then you MUST update # the upstream section in the nginx config to match. @@ -58,7 +69,7 @@ Bluepill.application("discourse", :base_dir => ENV["HOME"] + '/.bluepill') do |a #debug instance # app.process("thin-debug") do |process| -# process.start_command = "bundle exec thin start -e development -t 0 -p 10040 -P #{rails_root}/tmp/pids/thin-debug.pid -d" +# process.start_command = "bundle exec thin start -e development -t 0 -p 10040 -P #{rails_root}/tmp/pids/thin-debug.pid -l #{rails_root}/log/thin-debug.log" -d" # process.pid_file = "#{rails_root}/tmp/pids/thin-debug.pid" # process.start_grace_time = 30.seconds # process.stop_grace_time = 10.seconds @@ -73,7 +84,7 @@ Bluepill.application("discourse", :base_dir => ENV["HOME"] + '/.bluepill') do |a app.process("sidekiq-worker") do |process| pidfile = "#{rails_root}/tmp/pids/sidekiq-worker.pid" - process.start_command = "/usr/bin/env PIDFILE=#{pidfile} RAILS_ENV=#{rails_env} bundle exec sidekiq" + process.start_command = "/usr/bin/env PIDFILE=#{pidfile} RAILS_ENV=#{rails_env} bundle exec sidekiq -L #{rails_root}/log/sidekiq.log" process.pid_file = pidfile process.start_grace_time = 30.seconds process.stop_grace_time = 10.seconds @@ -87,7 +98,7 @@ Bluepill.application("discourse", :base_dir => ENV["HOME"] + '/.bluepill') do |a app.process("clockwork") do |process| pidfile = "#{rails_root}/tmp/pids/clockwork.pid" - process.start_command = "/usr/bin/env RAILS_ENV=#{rails_env} bundle exec clockwork config/clock.rb" + process.start_command = "/usr/bin/env CLOCK_LOG=#{rails_root}/log/clockwork.log RAILS_ENV=#{rails_env} bundle exec clockwork config/clock.rb" process.pid_file = pidfile process.start_grace_time = 30.seconds process.stop_grace_time = 10.seconds diff --git a/config/locales/server.en.yml b/config/locales/server.en.yml index 274f17112..99772bf69 100644 --- a/config/locales/server.en.yml +++ b/config/locales/server.en.yml @@ -446,7 +446,7 @@ en: ninja_edit_window: "Number of seconds after posting where edits do not create a new version" max_image_width: "Maximum allowed width of images in a post" category_featured_topics: "Number of topics displayed per category in the /categories page" - add_rel_nofollow_to_user_content: "Add rel nofollow to all submitted user content, except for internal links (including parent domains) changing this requires you update all your baked markdown (rake posts:rebake)" + add_rel_nofollow_to_user_content: "Add rel nofollow to all submitted user content, except for internal links (including parent domains) changing this requires you update all your baked markdown with: \"rake posts:rebake\"" exclude_rel_nofollow_domains: "A comma delimited list of domains where nofollow is not added (tld.com will automatically allow sub.tld.com as well)" post_excerpt_maxlength: "Maximum length in chars of a post's excerpt" diff --git a/config/logrotate.conf b/config/logrotate.conf new file mode 100644 index 000000000..1b29b530e --- /dev/null +++ b/config/logrotate.conf @@ -0,0 +1,38 @@ +# rotate log files daily, keep 32 days +daily +rotate 32 + +# create new (empty) log files after rotating old ones +create + +compress +missingok +create 0644 + + +/home/discourse/discourse/log/clockwork.log +{ + postrotate + pkill -f clockwork -HUP + endscript +} +/home/discourse/discourse/log/sidekiq.log +{ + postrotate + pkill -f sidekiq -USR2 + endscript +} +/home/discourse/discourse/log/production.log +{ + # ???? +} +/home/discourse/discourse/log/production_errors.log +{ + # production_errors is opened on demand, no need to reopen +} +/home/discourse/discourse/log/thin*.log +{ + postrotate + pkill -f 'thin server' -USR1 + endscript +} diff --git a/config/nginx.sample.conf b/config/nginx.sample.conf index f9fe3bbdf..7e193f948 100644 --- a/config/nginx.sample.conf +++ b/config/nginx.sample.conf @@ -21,6 +21,13 @@ server { location / { root /home/discourse/discourse/public; + location ~ \.(jpe?g|png|gif)$ { + valid_referers none blocked mysite.com *.mysite.com; + if ($invalid_referer) { + return 403; + } + } + location ~ ^/t\/[0-9]+\/[0-9]+\/avatar { expires 1d; add_header Cache-Control public; diff --git a/docs/INSTALL-ubuntu.md b/docs/INSTALL-ubuntu.md index 4f686fa84..245c3c981 100644 --- a/docs/INSTALL-ubuntu.md +++ b/docs/INSTALL-ubuntu.md @@ -204,31 +204,31 @@ Configure discourse: cp discourse.pill.sample discourse.pill cp environments/production.rb.sample environments/production.rb -Edit discourse/config/database.yml +Edit ~/discourse/config/database.yml - change production db name if appropriate - change username/password if appropriate - set db_id if using multisite - change `host_names` to the name you'll use to access the discourse site -Edit discourse/config/redis.yml +Edit ~/discourse/config/redis.yml - no changes if this is the only application using redis, but have a look -Edit discourse/config/discourse.pill +Edit ~/discourse/config/discourse.pill - change application name from 'discourse' if necessary - Ensure appropriate Bluepill.application line is uncommented - search for "host to run on" and change to current hostname - note: clockwork should run on only one host -Edit discourse/config/initializers/secret_token.rb +Edit ~/discourse/config/initializers/secret_token.rb - uncomment secret_token line -- replace SET_SECRET_HERE with secret output from 'rake secret' command in discourse directory +- replace SET_SECRET_HERE with secret output from 'RAILS_ENV=production rake secret' command in discourse directory - delete the lines below as per instructions in the file -Edit discourse/config/environments/production.rb +Edit ~/discourse/config/environments/production.rb - check settings, modify smtp settings if necessary - See http://meta.discourse.org/t/all-of-my-internal-users-show-as-coming-from-127-0-0-1/6607 if this will serve "internal" users @@ -281,9 +281,11 @@ Add the bluepill startup to crontab. # Run these commands as the discourse user crontab -e -Add the following line: +Add the following lines: @reboot RUBY_GC_MALLOC_LIMIT=90000000 RAILS_ROOT=~/discourse RAILS_ENV=production NUM_WEBS=4 /home/discourse/.rvm/bin/bootup_bluepill --no-privileged -c ~/.bluepill load ~/discourse/config/discourse.pill + 0 0 * * * /usr/sbin/logrotate ~/discourse/config/logrotate.conf + Note: in case of RVM system-wide installation RVM will be located in `/usr/local/rvm` directory instead of `/home/discourse/.rvm`, so update the line above respectively. diff --git a/docs/TROUBLESHOOTING-prod.md b/docs/TROUBLESHOOTING-prod.md new file mode 100644 index 000000000..463ce3c6f --- /dev/null +++ b/docs/TROUBLESHOOTING-prod.md @@ -0,0 +1,77 @@ +## Troubleshooting issues with Discourse environments + +Are you having trouble setting up Discourse? Here are some basic things to +check before reaching out to the community for help: + + +1. Are you running Ruby 1.9.3 or later? + + Discourse is designed for Ruby 1.9.3 or later. You can check your version by +typing `ruby -v` and checking the response. + + +2. Are you on Postgres 9.1 or later with HSTORE enabled? + + You can check your postgres version by typing `psql --version`. To see if hstore is + installed, open a session to postgres and type `\dx` and see if hstore is listed. + + +3. Have you run `bundle install`? + + We frequently update our dependencies to newer versions. It is a good idea to run + `bundle install` every time you check out Discourse, especially if it's been a while. + +4. Did you run `bundle update`? + + Don't. Running `bundle update` will download gem versions that we haven't tested with. + The Gemfile.lock has the gem versions that Discourse currently uses, so `bundle install` + will work. If you ran update, then you should uninstall the gems, run + `git checkout -- Gemfile.lock` and then run `bundle install`. + +5. Have you migrated your database? + + Our schema changes fairly frequently. After checking out the source code, you should + run `rake db:migrate` + + +6. Have you added the seed data? + + We depend on some basic seed data being present in the database. You should run + `rake db:seed_fu` to keep your database in sync. + + +7. Do the tests pass? + + If you are having other problems, it's useful to know if the test suite passes. You + can run it by first using `rake db:test:prepare` and then `rake spec`. If you + experience any failures, that's a bad sign! Our master branch should *always* pass + every test. + +8. Have you updated host_names in your database.yml? + + If links in emails have localhost in them, then you are still using the default host_names + value in database.yml. Update it to use your site's host name(s). + +9. Are you having problems bundling: + +``` +ArgumentError: invalid byte sequence in US-ASCII +An error occurred while installing active_model_serializers (0.7.0), and Bundler cannot continue. +Make sure that `gem install active_model_serializers -v '0.7.0'` succeeds before bundling. +``` + + Try this in console: + +``` +$ export LANG="en_US.UTF-8" +$ export LC_ALL="en_US.UTF-8" +``` + + And/or this in top of `Gemfile`: + +``` +if RUBY_VERSION =~ /1.9/ +Encoding.default_external = Encoding::UTF_8 +Encoding.default_internal = Encoding::UTF_8 +end +``` From 70be638f47b8504ab43351a7472fde35a623d0a8 Mon Sep 17 00:00:00 2001 From: Michael Brown Date: Tue, 18 Jun 2013 02:28:30 -0400 Subject: [PATCH 2/4] Update sample bluepill config and install docs --- config/discourse.pill.sample | 12 ++++++++---- docs/INSTALL-ubuntu.md | 25 ++++++++++++++++++++++--- 2 files changed, 30 insertions(+), 7 deletions(-) diff --git a/config/discourse.pill.sample b/config/discourse.pill.sample index a9afb4817..271f86777 100644 --- a/config/discourse.pill.sample +++ b/config/discourse.pill.sample @@ -32,9 +32,10 @@ Bluepill.application("discourse", :base_dir => ENV["HOME"] + '/.bluepill') do |a if bootup_bundle app.environment = `env -i BUNDLE_GEMFILE=#{rails_root}/Gemfile #{bootup_bundle} exec env`.lines.inject({}) do |env_hash,l| - kv = l.chomp.split('=',2) - env_hash[kv[0]] = kv[1] - env_hash + kv = l.chomp.split('=',2) + env_hash[kv[0]] = kv[1] + env_hash + end end app.environment ||= {} @@ -94,7 +95,10 @@ Bluepill.application("discourse", :base_dir => ENV["HOME"] + '/.bluepill') do |a process.daemonize = true end - if `hostname`.strip == "host to run on" + # Perhaps you want to run `hostname -f` here? It depends on your environment + # In any case, ensure the output is what you expect it to be on the system + # running clockwork! + if `hostname -s`.strip == "host to run on" app.process("clockwork") do |process| pidfile = "#{rails_root}/tmp/pids/clockwork.pid" diff --git a/docs/INSTALL-ubuntu.md b/docs/INSTALL-ubuntu.md index 245c3c981..d4d1bee0f 100644 --- a/docs/INSTALL-ubuntu.md +++ b/docs/INSTALL-ubuntu.md @@ -208,7 +208,7 @@ Edit ~/discourse/config/database.yml - change production db name if appropriate - change username/password if appropriate -- set db_id if using multisite +- set `db_id` if using multisite - change `host_names` to the name you'll use to access the discourse site Edit ~/discourse/config/redis.yml @@ -236,10 +236,22 @@ Initialize the database: # Run these commands as the discourse user # The database name here should match the production one in database.yml + cd ~/discourse createdb discourse_prod RUBY_GC_MALLOC_LIMIT=90000000 RAILS_ENV=production rake db:migrate RUBY_GC_MALLOC_LIMIT=90000000 RAILS_ENV=production rake assets:precompile +Not english? Set the default language as appropriate: + + # Run these commands as the discourse user + cd ~/discourse + RAILS_ENV=production bundle exec rails c + SiteSetting.default_locale = 'fr' + + # Not sure if your locale is supported? Check at the rails console: + LocaleSiteSetting.all_values + => ["cs", "da", "de", "en", "es", "fr", "id", "it", "nb_NO", "nl", "pseudo", "pt", "ru", "sv", "zh_CN", "zh_TW"] + ## nginx setup # Run these commands as your normal login (e.g. "michael") @@ -248,7 +260,12 @@ Initialize the database: Edit /etc/nginx/nginx.conf: - add: `server_names_hash_bucket_size 64;` to the `http` section -- uncomment: `gzip on;` + +If discourse will be the only site served by nginx, disable the nginx default +site: + +- `sudo mv /etc/nginx/conf.d/default.conf /etc/nginx/conf.d/default.conf.disabled` +- Otherwise, only `server_name`s configured below in `discourse.conf` will be passed to Discourse. Edit /etc/nginx/conf.d/discourse.conf @@ -284,11 +301,13 @@ Add the bluepill startup to crontab. Add the following lines: @reboot RUBY_GC_MALLOC_LIMIT=90000000 RAILS_ROOT=~/discourse RAILS_ENV=production NUM_WEBS=4 /home/discourse/.rvm/bin/bootup_bluepill --no-privileged -c ~/.bluepill load ~/discourse/config/discourse.pill - 0 0 * * * /usr/sbin/logrotate ~/discourse/config/logrotate.conf Note: in case of RVM system-wide installation RVM will be located in `/usr/local/rvm` directory instead of `/home/discourse/.rvm`, so update the line above respectively. +## Log rotation setup + 0 0 * * * /usr/sbin/logrotate ~/discourse/config/logrotate.conf + Congratulations! You've got Discourse installed and running! Now make yourself an administrator account. Browse to your discourse instance From b0d676f2f2e033a2593467ddf6dd370dc4204fb9 Mon Sep 17 00:00:00 2001 From: Michael Brown Date: Tue, 18 Jun 2013 03:04:01 -0400 Subject: [PATCH 3/4] fixes in logrotate.conf --- config/clock.rb | 1 + config/logrotate.conf | 8 +++++--- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/config/clock.rb b/config/clock.rb index 7183123f5..2674495f0 100644 --- a/config/clock.rb +++ b/config/clock.rb @@ -7,6 +7,7 @@ require_relative 'environment' def setup_log Clockwork.configure do |config| + config[:logger].close config[:logger] = Logger.new(ENV["CLOCK_LOG"]) end if ENV["CLOCK_LOG"] end diff --git a/config/logrotate.conf b/config/logrotate.conf index 1b29b530e..25d50d008 100644 --- a/config/logrotate.conf +++ b/config/logrotate.conf @@ -13,13 +13,15 @@ create 0644 /home/discourse/discourse/log/clockwork.log { postrotate - pkill -f clockwork -HUP + pkill -HUP -f clockwork endscript } /home/discourse/discourse/log/sidekiq.log { + # Should we just kill sidekiq? + # https://github.com/mperham/sidekiq/issues/1008 postrotate - pkill -f sidekiq -USR2 + pkill -USR2 -f sidekiq endscript } /home/discourse/discourse/log/production.log @@ -33,6 +35,6 @@ create 0644 /home/discourse/discourse/log/thin*.log { postrotate - pkill -f 'thin server' -USR1 + pkill -USR1 -f 'thin server' endscript } From d80b33c290ee4ae816863fbf8c7cf4a5bf82445a Mon Sep 17 00:00:00 2001 From: Michael Brown Date: Tue, 18 Jun 2013 03:05:12 -0400 Subject: [PATCH 4/4] Disable log rotation for now --- docs/INSTALL-ubuntu.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/INSTALL-ubuntu.md b/docs/INSTALL-ubuntu.md index d4d1bee0f..799c16e3a 100644 --- a/docs/INSTALL-ubuntu.md +++ b/docs/INSTALL-ubuntu.md @@ -306,7 +306,9 @@ Add the following lines: Note: in case of RVM system-wide installation RVM will be located in `/usr/local/rvm` directory instead of `/home/discourse/.rvm`, so update the line above respectively. ## Log rotation setup - 0 0 * * * /usr/sbin/logrotate ~/discourse/config/logrotate.conf + + # Disabled for now - log rotation isn't *quite* complete + #0 0 * * * /usr/sbin/logrotate ~/discourse/config/logrotate.conf Congratulations! You've got Discourse installed and running!